From dec56f1fcc4b05fb7fe8413826654bb5f3caf0c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Mar 2026 15:25:27 +0000 Subject: [PATCH 1/2] feat(cli): expose CLI features as a typed programmatic API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new `packages/cli/src/api/` module that wraps the core CLI operations (create, init, refresh, status) in typed, process-safe async functions: - `createDocset(params, options?)` – validates preset params and writes a new docset entry to .knowledge/config.yaml - `initDocset(params)` – downloads / symlinks all sources for a docset; supports optional path discovery and progress callbacks - `refreshDocsets(params?)` – pulls latest content for one or all docsets, returning structured per-docset results - `getStatus(params?)` – returns structured status for every docset without any console output Key design decisions: - No `process.exit()` – callers receive thrown errors instead - No chalk/console output – pure return values; CLI commands remain the only place that prints to stdout - Full TypeScript types for every parameter and return value, exported from `src/exports.ts` for library consumers - `cwd` option on every function so callers can control which .knowledge/config.yaml is used Refactor the four CLI command files to delegate to the new API functions, removing ~800 lines of duplicated business logic from the command handlers. https://claude.ai/code/session_01SExqFNVoBLQCCAbgAY6LXo --- .../cli/src/__tests__/cli-integration.test.ts | 44 +- packages/cli/src/api/create.ts | 171 ++++++ packages/cli/src/api/index.ts | 58 ++ packages/cli/src/api/init.ts | 73 +++ packages/cli/src/api/refresh.ts | 437 +++++++++++++++ packages/cli/src/api/status.ts | 148 ++++++ packages/cli/src/api/types.ts | 141 +++++ packages/cli/src/commands/create.ts | 179 +------ packages/cli/src/commands/init.ts | 71 +-- packages/cli/src/commands/refresh.ts | 502 +----------------- packages/cli/src/commands/status.ts | 262 ++------- packages/cli/src/exports.ts | 8 +- 12 files changed, 1141 insertions(+), 953 deletions(-) create mode 100644 packages/cli/src/api/create.ts create mode 100644 packages/cli/src/api/index.ts create mode 100644 packages/cli/src/api/init.ts create mode 100644 packages/cli/src/api/refresh.ts create mode 100644 packages/cli/src/api/status.ts create mode 100644 packages/cli/src/api/types.ts diff --git a/packages/cli/src/__tests__/cli-integration.test.ts b/packages/cli/src/__tests__/cli-integration.test.ts index d3825bd..0da9032 100644 --- a/packages/cli/src/__tests__/cli-integration.test.ts +++ b/packages/cli/src/__tests__/cli-integration.test.ts @@ -63,48 +63,20 @@ docsets: }); describe("Status Command", () => { - it("should show status with no initialized docsets", async () => { - // Change to test directory and run status command + it("should show status with no initialized docsets", () => { const originalCwd = process.cwd(); - const originalArgv = process.argv; process.chdir(testDir); try { - // Mock process.argv for status command - process.argv = ["node", "cli.js", "status"]; - - // Import and run CLI function directly - const { runCli } = await import("../cli.js"); - - // Capture console output - let output = ""; - const originalLog = console.log; - const originalError = console.error; - const originalWarn = console.warn; - const originalInfo = console.info; - - const capture = (...args: any[]) => { - output += args.join(" ") + "\n"; - }; - - console.log = capture; - console.error = capture; - console.warn = capture; - console.info = capture; - - await runCli(); - - // Restore console methods - console.log = originalLog; - console.error = originalError; - console.warn = originalWarn; - console.info = originalInfo; + const output = execSync(`node ${cliPath} status`, { + encoding: "utf8", + timeout: 5000, + }); expect(output).toContain("Agentic Knowledge Status"); - expect(output).toContain("Found 2 docset(s) with web sources"); + expect(output).toContain("Found 2 docset(s)"); } finally { process.chdir(originalCwd); - process.argv = originalArgv; } }); @@ -209,8 +181,8 @@ docsets: [] }); expect(output).toContain("Agentic Knowledge Refresh"); - expect(output).toContain("Found 2 docset(s) to refresh"); - expect(output).toContain("test-docset, unsupported-source-docset"); + expect(output).toContain("test-docset"); + expect(output).toContain("unsupported-source-docset"); } finally { process.chdir(originalCwd); } diff --git a/packages/cli/src/api/create.ts b/packages/cli/src/api/create.ts new file mode 100644 index 0000000..87912b4 --- /dev/null +++ b/packages/cli/src/api/create.ts @@ -0,0 +1,171 @@ +/** + * Typed API for creating new docsets. + */ + +import { promises as fs } from "node:fs"; +import * as path from "node:path"; +import { URL } from "node:url"; +import { ConfigManager } from "@codemcp/knowledge-core"; +import type { DocsetConfig } from "@codemcp/knowledge-core"; +import type { + CreateDocsetParams, + CreateDocsetOptions, + CreateDocsetResult, +} from "./types.js"; + +/** + * Create a new docset entry in the knowledge configuration. + * + * Validates the parameters for the chosen preset, creates the + * `.knowledge/config.yaml` if it doesn't exist yet, and appends the new + * docset to it. + * + * @throws {Error} when validation fails or the docset ID already exists. + */ +export async function createDocset( + params: CreateDocsetParams, + options?: CreateDocsetOptions, +): Promise { + const cwd = options?.cwd ?? process.cwd(); + const configManager = new ConfigManager(); + + const configExists = await configManager.configExists(cwd); + let config: { version: string; docsets: DocsetConfig[] }; + let configPath: string; + let configCreated = false; + + if (!configExists) { + configPath = path.join(cwd, ".knowledge", "config.yaml"); + config = { version: "1.0", docsets: [] }; + await fs.mkdir(path.dirname(configPath), { recursive: true }); + configCreated = true; + } else { + const loaded = await configManager.loadConfig(cwd); + config = loaded.config; + configPath = loaded.configPath; + } + + if (config.docsets.find((d) => d.id === params.id)) { + throw new Error(`Docset with ID '${params.id}' already exists`); + } + + let newDocset: DocsetConfig; + + if (params.preset === "git-repo") { + newDocset = buildGitRepoDocset(params); + } else if (params.preset === "local-folder") { + newDocset = await buildLocalFolderDocset(params); + } else if (params.preset === "archive") { + newDocset = await buildArchiveDocset(params); + } else { + throw new Error( + `Unknown preset: '${params.preset}'. Use 'git-repo', 'local-folder', or 'archive'`, + ); + } + + config.docsets.push(newDocset); + await configManager.saveConfig(config, configPath); + + return { docset: newDocset, configPath, configCreated }; +} + +function buildGitRepoDocset(params: CreateDocsetParams): DocsetConfig { + if (!params.url) { + throw new Error("url is required for the git-repo preset"); + } + if (!params.url.match(/^https?:\/\/.*\.git$|^git@.*\.git$/)) { + throw new Error("Invalid git URL format. Expected a URL ending with .git"); + } + return { + id: params.id, + name: params.name, + description: params.description ?? `Git repository: ${params.url}`, + sources: [ + { + url: params.url, + type: "git_repo" as const, + branch: params.branch ?? "main", + ...(params.paths !== undefined ? { paths: params.paths } : {}), + }, + ], + }; +} + +async function buildLocalFolderDocset( + params: CreateDocsetParams, +): Promise { + if (!params.path) { + throw new Error("path is required for the local-folder preset"); + } + const fullPath = path.resolve(params.path); + let stat; + try { + stat = await fs.stat(fullPath); + } catch { + throw new Error(`Path does not exist: ${params.path}`); + } + if (!stat.isDirectory()) { + throw new Error(`Path is not a directory: ${params.path}`); + } + return { + id: params.id, + name: params.name, + description: params.description ?? `Local documentation: ${params.path}`, + sources: [ + { + type: "local_folder", + paths: [params.path], + }, + ], + }; +} + +async function buildArchiveDocset( + params: CreateDocsetParams, +): Promise { + if (!params.path && !params.url) { + throw new Error("Either path or url is required for the archive preset"); + } + if (params.path) { + const fullPath = path.resolve(params.path); + let stat; + try { + stat = await fs.stat(fullPath); + } catch { + throw new Error(`Path does not exist or is invalid: ${params.path}`); + } + if (!stat.isFile()) { + throw new Error(`Path is not a file: ${params.path}`); + } + const lower = params.path.toLowerCase(); + if ( + !lower.endsWith(".zip") && + !lower.endsWith(".tar.gz") && + !lower.endsWith(".tgz") + ) { + throw new Error( + `Unsupported archive format. Expected .zip, .tar.gz or .tgz: ${params.path}`, + ); + } + } + if (params.url) { + try { + new URL(params.url); + } catch { + throw new Error(`Invalid URL format: ${params.url}`); + } + } + return { + id: params.id, + name: params.name, + description: params.description ?? `Archive: ${params.path ?? params.url}`, + sources: [ + { + type: "archive", + ...(params.path ? { path: params.path } : {}), + ...(params.url ? { url: params.url } : {}), + ...(params.paths ? { paths: params.paths } : {}), + }, + ], + }; +} diff --git a/packages/cli/src/api/index.ts b/packages/cli/src/api/index.ts new file mode 100644 index 0000000..0e41918 --- /dev/null +++ b/packages/cli/src/api/index.ts @@ -0,0 +1,58 @@ +/** + * Typed programmatic API for agentic-knowledge CLI features. + * + * Use these functions to interact with docsets from code without going through + * the CLI binary. All functions throw typed errors instead of calling + * `process.exit()`, return structured results instead of printing to stdout, + * and accept an optional `cwd` option so callers can control which + * `.knowledge/config.yaml` is used. + * + * @example + * ```ts + * import { createDocset, initDocset, getStatus } from "@codemcp/knowledge-cli"; + * + * // Add a new git-repo docset to the config + * const { docset, configPath } = await createDocset({ + * preset: "git-repo", + * id: "my-docs", + * name: "My Docs", + * url: "https://github.com/example/docs.git", + * }); + * + * // Download the sources + * const result = await initDocset({ docsetId: "my-docs" }); + * console.log(`Initialized ${result.totalFiles} files at ${result.localPath}`); + * + * // Query status + * const status = await getStatus(); + * console.log(status.docsets); + * ``` + */ + +export { createDocset } from "./create.js"; +export { initDocset } from "./init.js"; +export { refreshDocsets } from "./refresh.js"; +export { getStatus } from "./status.js"; +export type { + // create + DocsetPreset, + CreateDocsetParams, + CreateDocsetOptions, + CreateDocsetResult, + // init + InitDocsetParams, + InitDocsetApiResult, + // refresh + RefreshParams, + RefreshResult, + DocsetRefreshResult, + // status + StatusParams, + StatusResult, + DocsetStatusInfo, + DocsetSourceStatus, + // re-exported from dependencies + DocsetConfig, + SourceResult, + InitDocsetResult, +} from "./types.js"; diff --git a/packages/cli/src/api/init.ts b/packages/cli/src/api/init.ts new file mode 100644 index 0000000..9b8730e --- /dev/null +++ b/packages/cli/src/api/init.ts @@ -0,0 +1,73 @@ +/** + * Typed API for initialising docset sources. + */ + +import { + ConfigManager, + ensureKnowledgeGitignoreSync, + discoverDirectoryPatterns, +} from "@codemcp/knowledge-core"; +import { + initDocset as coreInitDocset, + type InitDocsetOptions, +} from "@codemcp/knowledge-content-loader"; +import type { InitDocsetParams, InitDocsetApiResult } from "./types.js"; + +/** + * Initialise all sources for a docset that is already defined in the config. + * + * Downloads / symlinks each source and writes the metadata files that the MCP + * server uses to serve the docset. + * + * @throws {Error} when the config cannot be found or the docset ID is unknown. + */ +export async function initDocset( + params: InitDocsetParams, +): Promise { + const { + docsetId, + force = false, + discoverPaths = false, + cwd = process.cwd(), + onSourceProgress, + } = params; + + const configManager = new ConfigManager(); + const { config, configPath } = await configManager.loadConfig(cwd); + + ensureKnowledgeGitignoreSync(configPath); + + const docset = config.docsets.find((d) => d.id === docsetId); + if (!docset) { + const available = config.docsets.map((d) => d.id).join(", "); + throw new Error( + `Docset '${docsetId}' not found in configuration. Available: ${available}`, + ); + } + + const coreOpts: InitDocsetOptions = { force }; + if (onSourceProgress !== undefined) { + coreOpts.onSourceProgress = onSourceProgress; + } + + const result = await coreInitDocset(docsetId, docset, configPath, coreOpts); + + if (result.alreadyInitialized) { + return { ...result }; + } + + if (discoverPaths) { + const allFiles = result.sourceResults.flatMap((r) => r.files); + if (allFiles.length > 0) { + const discovered = discoverDirectoryPatterns(allFiles); + try { + await configManager.updateDocsetPaths(docsetId, discovered); + } catch { + // Non-fatal: surface the discovered paths even if config update failed. + } + return { ...result, discoveredPaths: discovered }; + } + } + + return { ...result }; +} diff --git a/packages/cli/src/api/refresh.ts b/packages/cli/src/api/refresh.ts new file mode 100644 index 0000000..5b7ec0a --- /dev/null +++ b/packages/cli/src/api/refresh.ts @@ -0,0 +1,437 @@ +/** + * Typed API for refreshing docset sources. + */ + +import { promises as fs } from "node:fs"; +import * as path from "node:path"; +import { execSync } from "node:child_process"; +import { + findConfigPathSync, + loadConfigSync, + calculateLocalPath, + ensureKnowledgeGitignoreSync, +} from "@codemcp/knowledge-core"; +import { + ArchiveLoader, + WebSourceType, +} from "@codemcp/knowledge-content-loader"; +import type { + RefreshParams, + RefreshResult, + DocsetRefreshResult, +} from "./types.js"; + +interface DocsetMetadata { + docset_id: string; + docset_name: string; + initialized_at: string; + last_refreshed?: string; + total_files: number; + sources_count: number; +} + +interface SourceMetadata { + source_url: string; + source_type: string; + downloaded_at: string; + files_count: number; + files: string[]; + docset_id: string; + last_commit?: string; + content_hash?: string; +} + +/** + * Refresh sources for one or all docsets. + * + * For each docset the function pulls the latest content from every configured + * source and updates the metadata files on disk. Docsets that have not been + * initialised yet are silently skipped (check `skipped` / `skipReason` on + * the returned results). + * + * @throws {Error} when the config cannot be found or a named docset is unknown. + */ +export async function refreshDocsets( + params?: RefreshParams, +): Promise { + const { docsetId, force = false, cwd = process.cwd() } = params ?? {}; + + const configPath = findConfigPathSync(cwd); + if (!configPath) { + throw new Error( + "No configuration file found. Ensure .knowledge/config.yaml exists in the project.", + ); + } + + const config = loadConfigSync(configPath); + ensureKnowledgeGitignoreSync(configPath); + + const docsetsToRefresh = docsetId + ? config.docsets.filter((d) => d.id === docsetId) + : config.docsets.filter((d) => d.sources && d.sources.length > 0); + + if (docsetsToRefresh.length === 0) { + if (docsetId) { + const available = config.docsets + .filter((d) => d.sources && d.sources.length > 0) + .map((d) => d.id) + .join(", "); + throw new Error( + `Docset '${docsetId}' not found or has no sources. Available: ${available || "none"}`, + ); + } + return { docsets: [] }; + } + + const results: DocsetRefreshResult[] = []; + + for (const docset of docsetsToRefresh) { + const result = await refreshSingleDocset(docset, configPath, force); + results.push(result); + } + + return { docsets: results }; +} + +async function refreshSingleDocset( + docset: { id: string; sources?: unknown[] }, + configPath: string, + force: boolean, +): Promise { + const localPath = calculateLocalPath( + docset as Parameters[0], + configPath, + ); + const metadataPath = path.join(localPath, ".agentic-metadata.json"); + + // Must be initialised first. + let metadata: DocsetMetadata | null = null; + try { + metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); + } catch { + return { + docsetId: docset.id, + success: true, + skipped: true, + skipReason: "Not initialized — run init first", + }; + } + + // Skip if refreshed very recently (< 1 h) and force is not set. + if (!force && metadata) { + const lastRefresh = metadata.last_refreshed ?? metadata.initialized_at; + const hoursSince = + (Date.now() - new Date(lastRefresh).getTime()) / (1000 * 60 * 60); + if (hoursSince < 1) { + return { + docsetId: docset.id, + success: true, + skipped: true, + skipReason: `Recently refreshed (${Math.round(hoursSince * 60)} minutes ago)`, + }; + } + } + + // Back up metadata so we can restore on failure. + const backupPath = path.join(localPath, ".agentic-metadata.backup.json"); + await fs.copyFile(metadataPath, backupPath); + + try { + let totalFiles = 0; + let sourcesRefreshed = 0; + const sources = (docset as { sources?: unknown[] }).sources ?? []; + + for (const [index, source] of sources.entries()) { + const src = source as { + type: string; + url?: string; + path?: string; + paths?: string[]; + branch?: string; + }; + + if (src.type === "git_repo") { + const sm = await refreshGitSource( + src, + localPath, + index, + docset.id, + force, + ); + totalFiles += sm.files_count; + sourcesRefreshed++; + } else if (src.type === "archive") { + const sm = await refreshArchiveSource( + src, + localPath, + index, + docset.id, + force, + ); + totalFiles += sm.files_count; + sourcesRefreshed++; + } + // local_folder sources do not need refreshing (they are symlinked). + } + + const updatedMetadata: DocsetMetadata = { + docset_id: metadata!.docset_id, + docset_name: metadata!.docset_name, + initialized_at: metadata!.initialized_at, + last_refreshed: new Date().toISOString(), + total_files: totalFiles, + sources_count: sources.length, + }; + + await fs.writeFile(metadataPath, JSON.stringify(updatedMetadata, null, 2)); + await fs.unlink(backupPath); + + return { + docsetId: docset.id, + success: true, + skipped: false, + totalFiles, + sourcesRefreshed, + }; + } catch (error) { + // Attempt to restore from backup. + try { + await fs.copyFile(backupPath, metadataPath); + await fs.unlink(backupPath); + } catch { + // Ignore restore errors. + } + return { + docsetId: docset.id, + success: false, + skipped: false, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +async function refreshGitSource( + source: { url?: string; branch?: string; paths?: string[] }, + localPath: string, + index: number, + docsetId: string, + force: boolean, +): Promise { + const sourceMetadataPath = path.join( + localPath, + `.agentic-source-${index}.json`, + ); + let existing: SourceMetadata | null = null; + try { + existing = JSON.parse(await fs.readFile(sourceMetadataPath, "utf8")); + } catch { + // No prior metadata — full refresh. + } + + const tempDir = path.join(localPath, ".tmp", `git-refresh-${Date.now()}`); + await fs.mkdir(tempDir, { recursive: true }); + + try { + const branch = source.branch ?? "main"; + execSync( + `git clone --depth 1 --branch ${branch} ${source.url} ${tempDir}`, + { stdio: "pipe", timeout: 60000 }, + ); + + const latestCommit = execSync("git rev-parse HEAD", { + cwd: tempDir, + encoding: "utf8", + }).trim(); + + if (!force && existing) { + const lastCommit = (existing as SourceMetadata & { last_commit?: string }) + .last_commit; + if (lastCommit === latestCommit) { + const updated: SourceMetadata = { + ...existing, + downloaded_at: new Date().toISOString(), + }; + await fs.writeFile( + sourceMetadataPath, + JSON.stringify(updated, null, 2), + ); + return updated; + } + } + + // Remove old files tracked by this source. + if (existing) { + for (const file of existing.files) { + try { + await fs.unlink(path.join(localPath, file)); + } catch { + // Already deleted. + } + } + } + + const filesToCopy: string[] = []; + const paths = source.paths ?? []; + + if (paths.length > 0) { + for (const relPath of paths) { + const srcPath = path.join(tempDir, relPath); + const dstPath = path.join(localPath, relPath); + try { + const stat = await fs.stat(srcPath); + if (stat.isDirectory()) { + filesToCopy.push( + ...(await copyDir(srcPath, dstPath)).map((f) => + path.join(relPath, f), + ), + ); + } else { + await fs.mkdir(path.dirname(dstPath), { recursive: true }); + await fs.copyFile(srcPath, dstPath); + filesToCopy.push(relPath); + } + } catch { + // Skip missing paths. + } + } + } else { + const mdFiles = await findMarkdownFiles(tempDir); + for (const file of mdFiles) { + const rel = path.relative(tempDir, file); + const dst = path.join(localPath, rel); + await fs.mkdir(path.dirname(dst), { recursive: true }); + await fs.copyFile(file, dst); + filesToCopy.push(rel); + } + } + + const metadata: SourceMetadata & { last_commit: string } = { + source_url: source.url ?? "", + source_type: "git_repo", + downloaded_at: new Date().toISOString(), + files_count: filesToCopy.length, + files: filesToCopy, + docset_id: docsetId, + last_commit: latestCommit, + }; + + await fs.writeFile(sourceMetadataPath, JSON.stringify(metadata, null, 2)); + return metadata; + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +} + +async function refreshArchiveSource( + source: { url?: string; path?: string; paths?: string[] }, + localPath: string, + index: number, + docsetId: string, + force: boolean, +): Promise { + const sourceMetadataPath = path.join( + localPath, + `.agentic-source-${index}.json`, + ); + let existing: SourceMetadata | null = null; + try { + existing = JSON.parse(await fs.readFile(sourceMetadataPath, "utf8")); + } catch { + // No prior metadata. + } + + const sourceUrl = source.url ?? source.path ?? ""; + const loader = new ArchiveLoader(); + const webSourceConfig = { + url: sourceUrl, + type: WebSourceType.ARCHIVE, + options: { paths: source.paths ?? [] }, + }; + + if (!force && existing) { + try { + const currentId = await loader.getContentId(webSourceConfig); + const lastHash = (existing as SourceMetadata & { content_hash?: string }) + .content_hash; + if (lastHash === currentId) { + const updated: SourceMetadata = { + ...existing, + downloaded_at: new Date().toISOString(), + }; + await fs.writeFile( + sourceMetadataPath, + JSON.stringify(updated, null, 2), + ); + return updated; + } + } catch { + // Cannot check — do full refresh. + } + } + + if (existing) { + for (const file of existing.files) { + try { + await fs.unlink(path.join(localPath, file)); + } catch { + // Already deleted. + } + } + } + + const loadResult = await loader.load(webSourceConfig, localPath); + if (!loadResult.success) { + throw new Error(`Archive refresh failed: ${loadResult.error}`); + } + + const metadata: SourceMetadata & { content_hash?: string } = { + source_url: sourceUrl, + source_type: "archive", + downloaded_at: new Date().toISOString(), + files_count: loadResult.files.length, + files: loadResult.files, + docset_id: docsetId, + content_hash: loadResult.contentHash, + }; + + await fs.writeFile(sourceMetadataPath, JSON.stringify(metadata, null, 2)); + return metadata; +} + +async function findMarkdownFiles(dir: string): Promise { + const results: string[] = []; + async function walk(current: string) { + const entries = await fs.readdir(current); + for (const entry of entries) { + if (entry.startsWith(".git")) continue; + const full = path.join(current, entry); + const stat = await fs.stat(full); + if (stat.isDirectory()) { + await walk(full); + } else if (entry.endsWith(".md") || entry.endsWith(".mdx")) { + results.push(full); + } + } + } + await walk(dir); + return results; +} + +async function copyDir(src: string, dst: string): Promise { + const files: string[] = []; + await fs.mkdir(dst, { recursive: true }); + for (const entry of await fs.readdir(src)) { + const srcEntry = path.join(src, entry); + const dstEntry = path.join(dst, entry); + const stat = await fs.stat(srcEntry); + if (stat.isDirectory()) { + files.push( + ...(await copyDir(srcEntry, dstEntry)).map((f) => path.join(entry, f)), + ); + } else { + await fs.copyFile(srcEntry, dstEntry); + files.push(entry); + } + } + return files; +} diff --git a/packages/cli/src/api/status.ts b/packages/cli/src/api/status.ts new file mode 100644 index 0000000..f33cd3a --- /dev/null +++ b/packages/cli/src/api/status.ts @@ -0,0 +1,148 @@ +/** + * Typed API for querying docset status. + */ + +import { promises as fs } from "node:fs"; +import * as path from "node:path"; +import { + findConfigPathSync, + loadConfigSync, + calculateLocalPath, +} from "@codemcp/knowledge-core"; +import type { + StatusParams, + StatusResult, + DocsetStatusInfo, + DocsetSourceStatus, +} from "./types.js"; + +interface RawDocsetMetadata { + docset_id: string; + docset_name: string; + initialized_at: string; + last_refreshed?: string; + total_files: number; + sources_count: number; +} + +interface RawSourceMetadata { + source_url: string; + source_type: string; + downloaded_at: string; + files_count: number; + files: string[]; + docset_id: string; + last_commit?: string; + content_hash?: string; +} + +/** + * Return the initialisation status for every docset in the configuration. + * + * @throws {Error} when no configuration file can be found. + */ +export async function getStatus(params?: StatusParams): Promise { + const cwd = params?.cwd ?? process.cwd(); + const configPath = findConfigPathSync(cwd); + if (!configPath) { + throw new Error( + "No configuration file found. Ensure .knowledge/config.yaml exists in the project.", + ); + } + + const config = loadConfigSync(configPath); + const docsets: DocsetStatusInfo[] = []; + + for (const docset of config.docsets) { + docsets.push(await getDocsetStatus(docset, configPath)); + } + + return { configPath, docsets }; +} + +async function getDocsetStatus( + docset: { + id: string; + name: string; + description?: string; + sources?: unknown[]; + }, + configPath: string, +): Promise { + try { + const localPath = calculateLocalPath( + docset as Parameters[0], + configPath, + ); + const metadataPath = path.join(localPath, ".agentic-metadata.json"); + + let rawMetadata: RawDocsetMetadata | null = null; + try { + rawMetadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); + } catch { + const notInit: DocsetStatusInfo = { + id: docset.id, + name: docset.name, + initialized: false, + sources: [], + }; + if (docset.description !== undefined) + notInit.description = docset.description; + return notInit; + } + + const sources: DocsetSourceStatus[] = []; + for (let i = 0; i < (docset.sources?.length ?? 0); i++) { + try { + const srcPath = path.join(localPath, `.agentic-source-${i}.json`); + const raw: RawSourceMetadata = JSON.parse( + await fs.readFile(srcPath, "utf8"), + ); + const entry: DocsetSourceStatus = { + sourceUrl: raw.source_url, + sourceType: raw.source_type, + downloadedAt: raw.downloaded_at, + filesCount: raw.files_count, + }; + if (raw.last_commit !== undefined) entry.lastCommit = raw.last_commit; + if (raw.content_hash !== undefined) + entry.contentHash = raw.content_hash; + sources.push(entry); + } catch { + // Source metadata missing — skip. + } + } + + const initialized: DocsetStatusInfo = { + id: docset.id, + name: docset.name, + initialized: true, + sources, + }; + if (docset.description !== undefined) + initialized.description = docset.description; + if (rawMetadata) { + const meta: DocsetStatusInfo["metadata"] = { + initializedAt: rawMetadata.initialized_at, + totalFiles: rawMetadata.total_files, + sourcesCount: rawMetadata.sources_count, + }; + if (rawMetadata.last_refreshed !== undefined) { + meta.lastRefreshed = rawMetadata.last_refreshed; + } + initialized.metadata = meta; + } + return initialized; + } catch (error) { + const errInfo: DocsetStatusInfo = { + id: docset.id, + name: docset.name, + initialized: false, + sources: [], + error: error instanceof Error ? error.message : String(error), + }; + if (docset.description !== undefined) + errInfo.description = docset.description; + return errInfo; + } +} diff --git a/packages/cli/src/api/types.ts b/packages/cli/src/api/types.ts new file mode 100644 index 0000000..1aa39c9 --- /dev/null +++ b/packages/cli/src/api/types.ts @@ -0,0 +1,141 @@ +/** + * Typed API parameter and result types for programmatic use of CLI features. + */ + +import type { DocsetConfig } from "@codemcp/knowledge-core"; +import type { + SourceResult, + InitDocsetResult, +} from "@codemcp/knowledge-content-loader"; + +export type { DocsetConfig, SourceResult, InitDocsetResult }; + +// --------------------------------------------------------------------------- +// create +// --------------------------------------------------------------------------- + +export type DocsetPreset = "git-repo" | "local-folder" | "archive"; + +export interface CreateDocsetParams { + /** Preset type controlling which source type is created */ + preset: DocsetPreset; + /** Unique identifier for the new docset */ + id: string; + /** Human-readable display name */ + name: string; + /** Optional description */ + description?: string; + /** Git repository URL (git-repo) or remote archive URL (archive) */ + url?: string; + /** Local folder path (local-folder) or local archive file path (archive) */ + path?: string; + /** Git branch to clone (git-repo, defaults to "main") */ + branch?: string; + /** Specific sub-paths to extract/include from the source */ + paths?: string[]; +} + +export interface CreateDocsetOptions { + /** Working directory used to locate or create .knowledge/config.yaml (defaults to process.cwd()) */ + cwd?: string; +} + +export interface CreateDocsetResult { + /** The newly created docset configuration that was written to the config file */ + docset: DocsetConfig; + /** Absolute path of the config file that was written */ + configPath: string; + /** True when the .knowledge/config.yaml file was newly created by this call */ + configCreated: boolean; +} + +// --------------------------------------------------------------------------- +// init +// --------------------------------------------------------------------------- + +export interface InitDocsetParams { + /** ID of the docset to initialize (must exist in config) */ + docsetId: string; + /** Force re-initialization even if the docset is already present */ + force?: boolean; + /** + * After initialization, discover directory patterns from extracted files + * and update the config with them. + */ + discoverPaths?: boolean; + /** Working directory used to locate .knowledge/config.yaml (defaults to process.cwd()) */ + cwd?: string; + /** Called after each source is processed so callers can display progress */ + onSourceProgress?: (result: SourceResult) => void; +} + +export interface InitDocsetApiResult extends InitDocsetResult { + /** Patterns discovered from extracted files (only set when discoverPaths was true) */ + discoveredPaths?: string[]; +} + +// --------------------------------------------------------------------------- +// refresh +// --------------------------------------------------------------------------- + +export interface RefreshParams { + /** ID of a specific docset to refresh; omit to refresh all docsets */ + docsetId?: string; + /** Force refresh even if content appears unchanged */ + force?: boolean; + /** Working directory used to locate .knowledge/config.yaml (defaults to process.cwd()) */ + cwd?: string; +} + +export interface DocsetRefreshResult { + docsetId: string; + success: boolean; + /** True when the docset was intentionally skipped (e.g. recently refreshed, not initialized) */ + skipped: boolean; + skipReason?: string; + totalFiles?: number; + sourcesRefreshed?: number; + error?: string; +} + +export interface RefreshResult { + docsets: DocsetRefreshResult[]; +} + +// --------------------------------------------------------------------------- +// status +// --------------------------------------------------------------------------- + +export interface StatusParams { + /** Working directory used to locate .knowledge/config.yaml (defaults to process.cwd()) */ + cwd?: string; +} + +export interface DocsetSourceStatus { + sourceUrl: string; + sourceType: string; + downloadedAt: string; + filesCount: number; + lastCommit?: string; + contentHash?: string; +} + +export interface DocsetStatusInfo { + id: string; + name: string; + description?: string; + initialized: boolean; + metadata?: { + initializedAt: string; + lastRefreshed?: string; + totalFiles: number; + sourcesCount: number; + }; + sources: DocsetSourceStatus[]; + error?: string; +} + +export interface StatusResult { + configPath: string; + docsets: DocsetStatusInfo[]; +} diff --git a/packages/cli/src/commands/create.ts b/packages/cli/src/commands/create.ts index 020b2e3..85062d5 100644 --- a/packages/cli/src/commands/create.ts +++ b/packages/cli/src/commands/create.ts @@ -4,11 +4,7 @@ import { Command } from "commander"; import chalk from "chalk"; -import { promises as fs } from "node:fs"; -import * as path from "node:path"; -import { URL } from "node:url"; -import { ConfigManager } from "@codemcp/knowledge-core"; -import type { DocsetConfig } from "@codemcp/knowledge-core"; +import { createDocset } from "../api/create.js"; export const createCommand = new Command("create") .description("Create a new docset using presets") @@ -32,55 +28,24 @@ export const createCommand = new Command("create") try { console.log(chalk.blue("🚀 Creating new docset...")); - const configManager = new ConfigManager(); - - // Check if config exists, create if not - let config, configPath; - const configExists = await configManager.configExists(process.cwd()); - - if (!configExists) { - // Create initial config structure - configPath = path.join(process.cwd(), ".knowledge", "config.yaml"); - config = { - version: "1.0", - docsets: [], - }; + const { docset, configPath, configCreated } = await createDocset( + { + preset: options.preset, + id: options.id, + name: options.name, + description: options.description, + url: options.url, + path: options.path, + branch: options.branch, + }, + { cwd: process.cwd() }, + ); - // Ensure .knowledge directory exists - await fs.mkdir(path.dirname(configPath), { recursive: true }); + if (configCreated) { console.log(chalk.gray("📁 Created .knowledge directory")); - } else { - ({ config, configPath } = await configManager.loadConfig( - process.cwd(), - )); - } - - // Check if docset ID already exists - if (config.docsets.find((d) => d.id === options.id)) { - throw new Error(`Docset with ID '${options.id}' already exists`); - } - - let newDocset: DocsetConfig; - - if (options.preset === "git-repo") { - newDocset = await createGitRepoDocset(options); - } else if (options.preset === "local-folder") { - newDocset = await createLocalFolderDocset(options); - } else if (options.preset === "archive") { - newDocset = await createArchiveDocset(options); - } else { - throw new Error( - `Unknown preset: ${options.preset}. Use 'git-repo', 'local-folder', or 'archive'`, - ); } - // Add to config - config.docsets.push(newDocset); - await configManager.saveConfig(config, configPath); - - console.log( - chalk.green(`✅ Created docset '${options.id}' successfully`), - ); + console.log(chalk.green(`✅ Created docset '${docset.id}' successfully`)); console.log(chalk.gray(` Config saved to: ${configPath}`)); console.log( chalk.yellow( @@ -95,117 +60,3 @@ export const createCommand = new Command("create") process.exit(1); } }); - -async function createGitRepoDocset(options: any): Promise { - if (!options.url) { - throw new Error("--url is required for git-repo preset"); - } - - // Basic URL validation - if (!options.url.match(/^https?:\/\/.*\.git$|^git@.*\.git$/)) { - throw new Error("Invalid git URL format. Expected .git URL"); - } - - return { - id: options.id, - name: options.name, - description: options.description || `Git repository: ${options.url}`, - sources: [ - { - url: options.url, - type: "git_repo", - branch: options.branch, - paths: options.paths ? options.paths.split(",") : undefined, - }, - ], - }; -} - -async function createLocalFolderDocset(options: any): Promise { - if (!options.path) { - throw new Error("--path is required for local-folder preset"); - } - - // Validate path exists - const fullPath = path.resolve(options.path); - try { - const stat = await fs.stat(fullPath); - if (!stat.isDirectory()) { - throw new Error(`Path is not a directory: ${options.path}`); - } - } catch { - throw new Error(`Path does not exist: ${options.path}`); - } - - return { - id: options.id, - name: options.name, - description: options.description || `Local documentation: ${options.path}`, - sources: [ - { - type: "local_folder", - paths: [options.path], - }, - ], - }; -} - -async function createArchiveDocset(options: any): Promise { - if (!options.path && !options.url) { - throw new Error("Either --path or --url is required for archive preset"); - } - - // If path is provided, validate it exists - if (options.path) { - const fullPath = path.resolve(options.path); - try { - const stat = await fs.stat(fullPath); - if (!stat.isFile()) { - throw new Error(`Path is not a file: ${options.path}`); - } - const lowerPath = options.path.toLowerCase(); - if ( - !lowerPath.endsWith(".zip") && - !lowerPath.endsWith(".tar.gz") && - !lowerPath.endsWith(".tgz") - ) { - throw new Error( - `File is not a supported archive format (zip, tar.gz): ${options.path}`, - ); - } - } catch { - throw new Error(`Path does not exist or is invalid: ${options.path}`); - } - } - - // If URL is provided, validate it's a valid URL - if (options.url) { - try { - new URL(options.url); - } catch { - throw new Error(`Invalid URL format: ${options.url}`); - } - } - - const source: any = { - type: "archive", - }; - - if (options.path) { - source.path = options.path; - } - if (options.url) { - source.url = options.url; - } - if (options.paths) { - source.paths = options.paths.split(","); - } - - return { - id: options.id, - name: options.name, - description: - options.description || `Archive: ${options.path || options.url}`, - sources: [source], - }; -} diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 3970126..d586374 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -4,15 +4,8 @@ import { Command } from "commander"; import chalk from "chalk"; -import { - ConfigManager, - ensureKnowledgeGitignoreSync, - discoverDirectoryPatterns, -} from "@codemcp/knowledge-core"; -import { - initDocset, - type SourceResult, -} from "@codemcp/knowledge-content-loader"; +import { initDocset } from "../api/init.js"; +import type { SourceResult } from "@codemcp/knowledge-content-loader"; export const initCommand = new Command("init") .description("Initialize sources for a docset from configuration") @@ -32,27 +25,11 @@ export const initCommand = new Command("init") console.log(chalk.blue("🚀 Agentic Knowledge Integration Test")); try { - const configManager = new ConfigManager(); - const { config, configPath } = await configManager.loadConfig( - process.cwd(), - ); - - ensureKnowledgeGitignoreSync(configPath); - - const docset = config.docsets.find((d) => d.id === docsetId); - - if (!docset) { - throw new Error( - `Docset '${docsetId}' not found in configuration. Available: ${config.docsets.map((d) => d.id).join(", ")}`, - ); - } - - console.log(chalk.green(`✅ Found docset: ${docset.name}`)); - console.log(chalk.gray(`📝 Description: ${docset.description}`)); - console.log(chalk.gray(`🔗 Sources: ${docset.sources.length}`)); - - const result = await initDocset(docsetId, docset, configPath, { + const result = await initDocset({ + docsetId, force: options.force, + discoverPaths: options.discoverPaths, + cwd: process.cwd(), onSourceProgress: (sourceResult: SourceResult) => { const icon = sourceResult.type === "git_repo" @@ -75,37 +52,14 @@ export const initCommand = new Command("init") return; } - // Update configuration with discovered paths (only if --discover-paths flag used) - const allFiles = result.sourceResults.flatMap((r) => r.files); - if (allFiles.length > 0 && options.discoverPaths) { + if (result.discoveredPaths && result.discoveredPaths.length > 0) { + const shown = result.discoveredPaths.slice(0, 5); + const extra = result.discoveredPaths.length > 5 ? "..." : ""; console.log( - chalk.yellow( - `\n📝 Discovering directory patterns from extracted files...`, + chalk.green( + ` ✅ Updated config with discovered patterns: ${shown.join(", ")}${extra}`, ), ); - - const directoryPatterns = discoverDirectoryPatterns(allFiles); - - console.log( - chalk.gray( - ` Found ${allFiles.length} files → ${directoryPatterns.length} patterns`, - ), - ); - - try { - await configManager.updateDocsetPaths(docsetId, directoryPatterns); - console.log( - chalk.green( - ` ✅ Updated config with discovered patterns: ${directoryPatterns.slice(0, 5).join(", ")}${directoryPatterns.length > 5 ? "..." : ""}`, - ), - ); - } catch (configError) { - console.log( - chalk.yellow( - ` ⚠️ Could not update config: ${configError instanceof Error ? configError.message : String(configError)}`, - ), - ); - } } console.log( @@ -113,9 +67,6 @@ export const initCommand = new Command("init") ); console.log(chalk.gray(`📁 Location: ${result.localPath}`)); console.log(chalk.gray(`📄 Total files: ${result.totalFiles}`)); - console.log( - chalk.gray(`🔗 Sources processed: ${docset.sources.length}`), - ); } catch (error) { console.error(chalk.red("\n❌ Error:")); console.error( diff --git a/packages/cli/src/commands/refresh.ts b/packages/cli/src/commands/refresh.ts index de2d21e..015e264 100644 --- a/packages/cli/src/commands/refresh.ts +++ b/packages/cli/src/commands/refresh.ts @@ -4,38 +4,7 @@ import { Command } from "commander"; import chalk from "chalk"; -import ora from "ora"; -import { promises as fs } from "node:fs"; -import * as path from "node:path"; -import { execSync } from "node:child_process"; -import { - findConfigPathSync, - loadConfigSync, - calculateLocalPath, - ensureKnowledgeGitignoreSync, -} from "@codemcp/knowledge-core"; -import { - ArchiveLoader, - WebSourceType, -} from "@codemcp/knowledge-content-loader"; - -interface DocsetMetadata { - docset_id: string; - docset_name: string; - initialized_at: string; - last_refreshed?: string; - total_files: number; - sources_count: number; -} - -interface SourceMetadata { - source_url: string; - source_type: string; - downloaded_at: string; - files_count: number; - files: string[]; - docset_id: string; -} +import { refreshDocsets } from "../api/refresh.js"; export const refreshCommand = new Command("refresh") .description("Refresh sources for docsets") @@ -53,50 +22,37 @@ export const refreshCommand = new Command("refresh") console.log(chalk.blue("🔄 Agentic Knowledge Refresh")); try { - // Find and load configuration - const configPath = options.config || findConfigPathSync(process.cwd()); - if (!configPath) { - throw new Error( - "No configuration file found. Run this command from a directory with .knowledge/config.yaml", - ); - } - - console.log(chalk.gray(`📄 Loading config: ${configPath}`)); - const config = loadConfigSync(configPath); - - // Ensure .knowledge/.gitignore exists - ensureKnowledgeGitignoreSync(configPath); + const refreshParams: Parameters[0] = { + force: options.force, + cwd: process.cwd(), + }; + if (docsetId !== undefined) refreshParams.docsetId = docsetId; + const result = await refreshDocsets(refreshParams); - // Determine which docsets to refresh - const docsetsToRefresh = docsetId - ? config.docsets.filter((d) => d.id === docsetId) - : config.docsets.filter((d) => d.sources && d.sources.length > 0); + if (result.docsets.length === 0) { + console.log(chalk.yellow("⚠️ No docsets with web sources found.")); + return; + } - if (docsetsToRefresh.length === 0) { - if (docsetId) { - throw new Error( - `Docset '${docsetId}' not found or has no sources. Available docsets with sources: ${ - config.docsets - .filter((d) => d.sources && d.sources.length > 0) - .map((d) => d.id) - .join(", ") || "none" - }`, + for (const ds of result.docsets) { + if (ds.skipped) { + console.log( + chalk.yellow(`⏭️ ${ds.docsetId}: Skipped — ${ds.skipReason}`), + ); + } else if (ds.success) { + console.log( + chalk.green( + `✅ ${ds.docsetId}: Refreshed (${ds.totalFiles} files from ${ds.sourcesRefreshed} source(s))`, + ), ); } else { - console.log(chalk.yellow("⚠️ No docsets with web sources found.")); - return; + console.log(chalk.red(`❌ ${ds.docsetId}: Failed — ${ds.error}`)); } } - console.log( - chalk.green( - `✅ Found ${docsetsToRefresh.length} docset(s) to refresh: ${docsetsToRefresh.map((d) => d.id).join(", ")}`, - ), - ); - - // Refresh each docset - for (const docset of docsetsToRefresh) { - await refreshDocset(docset, configPath, options.force); + const failed = result.docsets.filter((d) => !d.success && !d.skipped); + if (failed.length > 0) { + process.exit(1); } console.log(chalk.green("\n🎉 All docsets refreshed successfully!")); @@ -109,411 +65,3 @@ export const refreshCommand = new Command("refresh") } }, ); - -async function refreshDocset( - docset: any, - configPath: string, - force: boolean, -): Promise { - const spinner = ora(`Refreshing ${docset.id}...`).start(); - - try { - const localPath = calculateLocalPath(docset, configPath); - - spinner.text = `Checking ${docset.id} metadata...`; - - // Check if docset has been initialized - const metadataPath = path.join(localPath, ".agentic-metadata.json"); - let metadata: DocsetMetadata | null = null; - - try { - const metadataContent = await fs.readFile(metadataPath, "utf8"); - metadata = JSON.parse(metadataContent); - } catch { - spinner.warn(`${docset.id}: Not initialized, use 'init' command first`); - return; - } - - // Check if forced or if we should check for updates - if (!force && metadata) { - const lastRefresh = metadata.last_refreshed || metadata.initialized_at; - const lastRefreshTime = new Date(lastRefresh); - const timeSinceRefresh = Date.now() - lastRefreshTime.getTime(); - const hoursSince = timeSinceRefresh / (1000 * 60 * 60); - - if (hoursSince < 1) { - spinner.succeed( - `${docset.id}: Recently refreshed (${Math.round(hoursSince * 60)} minutes ago), skipping`, - ); - return; - } - } - - spinner.text = `Refreshing ${docset.id} web sources...`; - - // Create backup of current metadata - const backupPath = path.join(localPath, `.agentic-metadata.backup.json`); - await fs.copyFile(metadataPath, backupPath); - - let totalFiles = 0; - const refreshedSources: SourceMetadata[] = []; - - // Process each source - for (const [index, source] of (docset.sources || []).entries()) { - spinner.text = `${docset.id}: Refreshing source ${index + 1}/${docset.sources.length}...`; - - if (source.type === "git_repo") { - const sourceFiles = await refreshGitSource( - source, - localPath, - index, - docset.id, - force, - ); - totalFiles += sourceFiles.files_count; - refreshedSources.push(sourceFiles); - } else if (source.type === "archive") { - const sourceFiles = await refreshArchiveSource( - source, - localPath, - index, - docset.id, - force, - ); - totalFiles += sourceFiles.files_count; - refreshedSources.push(sourceFiles); - } else { - console.log( - chalk.yellow( - ` ⚠️ Source type '${source.type}' not yet supported, skipping`, - ), - ); - } - } - - // Update metadata - if (!metadata) { - throw new Error("Metadata is null - this should not happen"); - } - - const updatedMetadata: DocsetMetadata = { - docset_id: metadata.docset_id, - docset_name: metadata.docset_name, - initialized_at: metadata.initialized_at, - last_refreshed: new Date().toISOString(), - total_files: totalFiles, - sources_count: docset.sources?.length || 0, - }; - - await fs.writeFile(metadataPath, JSON.stringify(updatedMetadata, null, 2)); - - // Remove backup if successful - await fs.unlink(backupPath); - - spinner.succeed( - `${docset.id}: Refreshed successfully (${totalFiles} files from ${refreshedSources.length} sources)`, - ); - } catch (error) { - spinner.fail( - `${docset.id}: Failed to refresh - ${error instanceof Error ? error.message : String(error)}`, - ); - - // Try to restore from backup - const backupPath = path.join( - calculateLocalPath(docset, configPath), - `.agentic-metadata.backup.json`, - ); - try { - const metadataPath = path.join( - calculateLocalPath(docset, configPath), - ".agentic-metadata.json", - ); - await fs.copyFile(backupPath, metadataPath); - await fs.unlink(backupPath); - console.log(chalk.gray(` Restored metadata from backup`)); - } catch { - // Backup restore failed, but don't throw - } - - throw error; - } -} - -async function refreshGitSource( - webSource: any, - localPath: string, - index: number, - docsetId: string, - force: boolean, -): Promise { - // Check existing source metadata - const sourceMetadataPath = path.join( - localPath, - `.agentic-source-${index}.json`, - ); - let existingSourceMetadata: SourceMetadata | null = null; - - try { - const content = await fs.readFile(sourceMetadataPath, "utf8"); - existingSourceMetadata = JSON.parse(content); - } catch { - // No existing metadata, will do full refresh - } - - // Create temp directory for cloning - const tempDir = path.join(localPath, ".tmp", `git-refresh-${Date.now()}`); - await fs.mkdir(tempDir, { recursive: true }); - - try { - // Clone repository - const options = webSource.options || {}; - const branch = (options as any).branch || "main"; - const paths = (options as any).paths || []; - - execSync( - `git clone --depth 1 --branch ${branch} ${webSource.url} ${tempDir}`, - { - stdio: "pipe", - timeout: 60000, - }, - ); - - // Get latest commit hash for change detection - const latestCommit = execSync("git rev-parse HEAD", { - cwd: tempDir, - encoding: "utf8", - }).trim(); - - // Check if we need to update (compare with last known commit if available) - if (!force && existingSourceMetadata) { - const lastCommit = (existingSourceMetadata as any).last_commit; - if (lastCommit === latestCommit) { - // No changes, update timestamp only - const updatedMetadata: SourceMetadata = { - ...existingSourceMetadata, - downloaded_at: new Date().toISOString(), - }; - - await fs.writeFile( - sourceMetadataPath, - JSON.stringify(updatedMetadata, null, 2), - ); - - return updatedMetadata; - } - } - - // Remove old files from this source (if we have metadata) - if (existingSourceMetadata) { - for (const file of existingSourceMetadata.files) { - const filePath = path.join(localPath, file); - try { - await fs.unlink(filePath); - } catch { - // File might already be deleted, ignore - } - } - } - - // Copy new files - const filesToCopy: string[] = []; - - if (paths.length > 0) { - // Copy specified paths - for (const relPath of paths) { - const sourcePath = path.join(tempDir, relPath); - const targetPath = path.join(localPath, relPath); - - try { - const stat = await fs.stat(sourcePath); - if (stat.isDirectory()) { - const dirFiles = await copyDirectory(sourcePath, targetPath); - filesToCopy.push(...dirFiles); - } else { - await fs.mkdir(path.dirname(targetPath), { recursive: true }); - await fs.copyFile(sourcePath, targetPath); - filesToCopy.push(relPath); - } - } catch (error) { - console.log( - chalk.yellow( - ` ⚠️ Skipping ${relPath}: ${error instanceof Error ? error.message : String(error)}`, - ), - ); - } - } - } else { - // Copy all markdown files - const allFiles = await findMarkdownFiles(tempDir); - for (const file of allFiles) { - const relativePath = path.relative(tempDir, file); - const targetPath = path.join(localPath, relativePath); - - await fs.mkdir(path.dirname(targetPath), { recursive: true }); - await fs.copyFile(file, targetPath); - filesToCopy.push(relativePath); - } - } - - // Create updated source metadata - const metadata: SourceMetadata & { last_commit: string } = { - source_url: webSource.url, - source_type: webSource.type, - downloaded_at: new Date().toISOString(), - files_count: filesToCopy.length, - files: filesToCopy, - docset_id: docsetId, - last_commit: latestCommit, - }; - - await fs.writeFile(sourceMetadataPath, JSON.stringify(metadata, null, 2)); - - return metadata; - } finally { - // Cleanup temp directory - await fs.rm(tempDir, { recursive: true, force: true }); - } -} - -async function refreshArchiveSource( - source: any, - localPath: string, - index: number, - docsetId: string, - force: boolean, -): Promise { - const sourceMetadataPath = path.join( - localPath, - `.agentic-source-${index}.json`, - ); - let existingSourceMetadata: SourceMetadata | null = null; - - try { - const content = await fs.readFile(sourceMetadataPath, "utf8"); - existingSourceMetadata = JSON.parse(content); - } catch { - // No existing metadata, will do full refresh - } - - const sourceUrl = source.url || source.path || ""; - const loader = new ArchiveLoader(); - const webSourceConfig = { - url: sourceUrl, - type: WebSourceType.ARCHIVE, - options: { - paths: source.paths || [], - }, - }; - - // Check if content has changed - if (!force && existingSourceMetadata) { - try { - const currentId = await loader.getContentId(webSourceConfig); - const lastHash = (existingSourceMetadata as any).content_hash; - if (lastHash === currentId) { - const updatedMetadata: SourceMetadata = { - ...existingSourceMetadata, - downloaded_at: new Date().toISOString(), - }; - await fs.writeFile( - sourceMetadataPath, - JSON.stringify(updatedMetadata, null, 2), - ); - return updatedMetadata; - } - } catch { - // Could not check, proceed with full refresh - } - } - - // Remove old files from this source (if we have metadata) - if (existingSourceMetadata) { - for (const file of existingSourceMetadata.files) { - const filePath = path.join(localPath, file); - try { - await fs.unlink(filePath); - } catch { - // File might already be deleted, ignore - } - } - } - - // Load content - const result = await loader.load(webSourceConfig, localPath); - - if (!result.success) { - throw new Error(`Archive refresh failed: ${result.error}`); - } - - const metadata: SourceMetadata = { - source_url: sourceUrl, - source_type: "archive", - downloaded_at: new Date().toISOString(), - files_count: result.files.length, - files: result.files, - docset_id: docsetId, - }; - - // Store content hash for future change detection - const metadataWithHash = { - ...metadata, - content_hash: result.contentHash, - }; - - await fs.writeFile( - sourceMetadataPath, - JSON.stringify(metadataWithHash, null, 2), - ); - - return metadata; -} - -// Reuse utility functions from init.ts -async function findMarkdownFiles(dir: string): Promise { - const files: string[] = []; - - async function scan(currentDir: string) { - const items = await fs.readdir(currentDir); - - for (const item of items) { - if (item.startsWith(".git")) continue; - - const fullPath = path.join(currentDir, item); - const stat = await fs.stat(fullPath); - - if (stat.isDirectory()) { - await scan(fullPath); - } else if (item.endsWith(".md") || item.endsWith(".mdx")) { - files.push(fullPath); - } - } - } - - await scan(dir); - return files; -} - -async function copyDirectory( - source: string, - target: string, -): Promise { - const files: string[] = []; - await fs.mkdir(target, { recursive: true }); - const items = await fs.readdir(source); - - for (const item of items) { - const sourcePath = path.join(source, item); - const targetPath = path.join(target, item); - const stat = await fs.stat(sourcePath); - - if (stat.isDirectory()) { - const subFiles = await copyDirectory(sourcePath, targetPath); - files.push(...subFiles.map((f) => path.join(item, f))); - } else { - await fs.copyFile(sourcePath, targetPath); - files.push(item); - } - } - - return files; -} diff --git a/packages/cli/src/commands/status.ts b/packages/cli/src/commands/status.ts index 3b52584..2af63e5 100644 --- a/packages/cli/src/commands/status.ts +++ b/packages/cli/src/commands/status.ts @@ -4,40 +4,8 @@ import { Command } from "commander"; import chalk from "chalk"; -import { promises as fs } from "node:fs"; -import * as path from "node:path"; -import { - findConfigPathSync, - loadConfigSync, - calculateLocalPath, -} from "@codemcp/knowledge-core"; - -interface DocsetMetadata { - docset_id: string; - docset_name: string; - initialized_at: string; - last_refreshed?: string; - total_files: number; - sources_count: number; -} - -interface SourceMetadata { - source_url: string; - source_type: string; - downloaded_at: string; - files_count: number; - files: string[]; - docset_id: string; - last_commit?: string; -} - -interface DocsetStatus { - docset: any; - initialized: boolean; - metadata: DocsetMetadata | null; - sources: SourceMetadata[]; - error?: string; -} +import { getStatus } from "../api/status.js"; +import type { DocsetStatusInfo } from "../api/types.js"; export const statusCommand = new Command("status") .description("Show status of web sources for docsets") @@ -47,56 +15,25 @@ export const statusCommand = new Command("status") try { console.log(chalk.blue("📊 Agentic Knowledge Status\n")); - // Find and load configuration - const configPath = options.config || findConfigPathSync(process.cwd()); - if (!configPath) { - throw new Error( - "No configuration file found. Run this command from a directory with .knowledge/config.yaml", - ); - } - - console.log(chalk.gray(`📄 Config: ${configPath}`)); - const config = loadConfigSync(configPath); - - // Find docsets with web sources - const webDocsets = config.docsets.filter( - (d) => d.sources && d.sources.length > 0, - ); + const result = await getStatus({ cwd: process.cwd() }); - if (webDocsets.length === 0) { - console.log(chalk.yellow("\n⚠️ No docsets with web sources found.")); + console.log(chalk.gray(`📄 Config: ${result.configPath}`)); - // Show all docsets for reference - if (config.docsets.length > 0) { - console.log(chalk.gray("\nAvailable docsets (local only):")); - for (const docset of config.docsets) { - console.log(chalk.gray(` • ${docset.id} - ${docset.name}`)); - } - } + if (result.docsets.length === 0) { + console.log(chalk.yellow("\n⚠️ No docsets configured.")); return; } console.log( - chalk.green( - `\n✅ Found ${webDocsets.length} docset(s) with web sources\n`, - ), + chalk.green(`\n✅ Found ${result.docsets.length} docset(s)\n`), ); - // Get status for each docset - const statuses: DocsetStatus[] = []; - for (const docset of webDocsets) { - const status = await getDocsetStatus(docset, configPath); - statuses.push(status); - } - - // Display summary - displaySummary(statuses); + displaySummary(result.docsets); - // Display detailed status if verbose if (options.verbose) { console.log(chalk.blue("\n📋 Detailed Status\n")); - for (const status of statuses) { - displayDetailedStatus(status); + for (const ds of result.docsets) { + displayDetailed(ds); } } } catch (error) { @@ -108,145 +45,65 @@ export const statusCommand = new Command("status") } }); -async function getDocsetStatus( - docset: any, - configPath: string, -): Promise { - try { - const localPath = calculateLocalPath(docset, configPath); - const metadataPath = path.join(localPath, ".agentic-metadata.json"); - - // Check if docset is initialized - let metadata: DocsetMetadata | null = null; - try { - const metadataContent = await fs.readFile(metadataPath, "utf8"); - metadata = JSON.parse(metadataContent); - } catch { - return { - docset, - initialized: false, - metadata: null, - sources: [], - }; - } - - // Load source metadata - const sources: SourceMetadata[] = []; - for (let i = 0; i < (docset.sources?.length || 0); i++) { - try { - const sourceMetadataPath = path.join( - localPath, - `.agentic-source-${i}.json`, - ); - const sourceContent = await fs.readFile(sourceMetadataPath, "utf8"); - const sourceMetadata = JSON.parse(sourceContent); - sources.push(sourceMetadata); - } catch { - // Source metadata missing - this might indicate an issue - } - } - - return { - docset, - initialized: true, - metadata, - sources, - }; - } catch (error) { - return { - docset, - initialized: false, - metadata: null, - sources: [], - error: error instanceof Error ? error.message : String(error), - }; - } -} - -function displaySummary(statuses: DocsetStatus[]) { +function displaySummary(docsets: DocsetStatusInfo[]) { console.log(chalk.blue("📈 Summary")); console.log("─".repeat(50)); - for (const status of statuses) { - const { docset, initialized, metadata, sources, error } = status; - - if (error) { + for (const ds of docsets) { + if (ds.error) { console.log( - `${chalk.red("❌")} ${chalk.bold(docset.id)} - ${chalk.red("Error: " + error)}`, + `${chalk.red("❌")} ${chalk.bold(ds.id)} - ${chalk.red("Error: " + ds.error)}`, ); continue; } - if (!initialized) { - console.log(`${chalk.bold(docset.id)} (${docset.name})`); + if (!ds.initialized) { + console.log(`${chalk.bold(ds.id)} (${ds.name})`); console.log( chalk.gray( - ` Not initialized | ${docset.sources?.length || 0} source(s) configured`, + ` Not initialized | ${ds.sources.length} source(s) loaded`, ), ); console.log(); console.log( - chalk.blue(` 💡 Run: npx @codemcp/knowledge init ${docset.id}`), - ); - continue; - } - - if (!metadata) { - console.log( - `${chalk.red("❌")} ${chalk.bold(docset.id)} - ${chalk.red("Metadata corrupted")}`, + chalk.blue(` 💡 Run: npx @codemcp/knowledge init ${ds.id}`), ); continue; } - // Format initialization date - const initDate = new Date(metadata.initialized_at); - const dateDisplay = initDate.toISOString().split("T")[0]; // YYYY-MM-DD format + const dateDisplay = ds.metadata + ? new Date(ds.metadata.initializedAt).toISOString().split("T")[0] + : "unknown"; - console.log(`${chalk.bold(docset.id)} (${docset.name})`); + console.log(`${chalk.bold(ds.id)} (${ds.name})`); console.log( chalk.gray( - ` Initialized | ${metadata.total_files} files | ${sources.length}/${metadata.sources_count} source(s) loaded`, + ` Initialized | ${ds.metadata?.totalFiles ?? 0} files | ${ds.sources.length}/${ds.metadata?.sourcesCount ?? 0} source(s) loaded`, ), ); console.log(chalk.gray(` Initialized: ${dateDisplay}`)); } } -function displayDetailedStatus(status: DocsetStatus) { - const { docset, initialized, metadata, sources, error } = status; - - console.log(chalk.bold(`🔸 ${docset.id} (${docset.name})`)); +function displayDetailed(ds: DocsetStatusInfo) { + console.log(chalk.bold(`🔸 ${ds.id} (${ds.name})`)); console.log("─".repeat(40)); - if (error) { - console.log(chalk.red(`❌ Error: ${error}`)); + if (ds.error) { + console.log(chalk.red(`❌ Error: ${ds.error}`)); console.log(); return; } - if (!initialized) { + if (!ds.initialized) { console.log(chalk.yellow("⚠️ Status: Not initialized")); console.log( - chalk.gray(`📝 Description: ${docset.description || "No description"}`), - ); - console.log( - chalk.gray(`🔗 Sources configured: ${docset.sources?.length || 0}`), + chalk.gray(`📝 Description: ${ds.description ?? "No description"}`), ); - - if (docset.sources && docset.sources.length > 0) { - console.log(chalk.gray(" Sources:")); - for (const [i, source] of docset.sources.entries()) { - console.log( - chalk.gray( - ` ${i + 1}. ${source.type === "git_repo" ? source.url : source.paths?.join(", ")} (${source.type})`, - ), - ); - } + if (ds.sources.length === 0) { console.log( chalk.blue( - "\n 💡 Run 'npx @codemcp/knowledge init " + - docset.id + - "' to initialize", + `\n 💡 Run 'npx @codemcp/knowledge init ${ds.id}' to initialize`, ), ); } @@ -254,62 +111,43 @@ function displayDetailedStatus(status: DocsetStatus) { return; } - if (!metadata) { - console.log(chalk.red("❌ Status: Metadata corrupted")); - console.log(); - return; - } - - // Display basic info console.log(chalk.green("✅ Status: Initialized")); console.log( - chalk.gray(`📝 Description: ${docset.description || "No description"}`), + chalk.gray(`📝 Description: ${ds.description ?? "No description"}`), ); - console.log(chalk.gray(`📄 Total files: ${metadata.total_files}`)); - console.log(chalk.gray(`🔗 Sources: ${metadata.sources_count}`)); - - // Display timing info - const initTime = new Date(metadata.initialized_at); - const lastRefresh = metadata.last_refreshed - ? new Date(metadata.last_refreshed) - : null; + console.log(chalk.gray(`📄 Total files: ${ds.metadata?.totalFiles ?? 0}`)); + console.log(chalk.gray(`🔗 Sources: ${ds.metadata?.sourcesCount ?? 0}`)); - console.log(chalk.gray(`📅 Initialized: ${initTime.toLocaleString()}`)); - if (lastRefresh) { + if (ds.metadata) { console.log( - chalk.gray(`🔄 Last refreshed: ${lastRefresh.toLocaleString()}`), + chalk.gray( + `📅 Initialized: ${new Date(ds.metadata.initializedAt).toLocaleString()}`, + ), ); + if (ds.metadata.lastRefreshed) { + console.log( + chalk.gray( + `🔄 Last refreshed: ${new Date(ds.metadata.lastRefreshed).toLocaleString()}`, + ), + ); + } } - // Display source details - if (sources.length > 0) { + if (ds.sources.length > 0) { console.log(chalk.gray("\n🔗 Sources:")); - for (const [i, source] of sources.entries()) { - const downloadTime = new Date(source.downloaded_at); + for (const [i, src] of ds.sources.entries()) { console.log( chalk.gray( - ` ${i + 1}. ${source.source_url} (${source.files_count} files, ${downloadTime.toLocaleString()})`, + ` ${i + 1}. ${src.sourceUrl} (${src.filesCount} files, ${new Date(src.downloadedAt).toLocaleString()})`, ), ); - if (source.last_commit) { + if (src.lastCommit) { console.log( - chalk.gray( - ` Last commit: ${source.last_commit.substring(0, 8)}`, - ), + chalk.gray(` Last commit: ${src.lastCommit.substring(0, 8)}`), ); } } } - // Display missing sources - const missingSources = (docset.sources?.length || 0) - sources.length; - if (missingSources > 0) { - console.log( - chalk.yellow( - `⚠️ ${missingSources} source(s) missing metadata - run refresh`, - ), - ); - } - console.log(); } diff --git a/packages/cli/src/exports.ts b/packages/cli/src/exports.ts index 5349bdb..9f853fb 100644 --- a/packages/cli/src/exports.ts +++ b/packages/cli/src/exports.ts @@ -1,7 +1,7 @@ /** - * CLI exports for agentic-knowledge + * Typed programmatic API for @codemcp/knowledge-cli. + * + * Import from this module to use CLI features without the binary. */ -export * from "./commands/init.js"; -export * from "./commands/refresh.js"; -export * from "./commands/status.js"; +export * from "./api/index.js"; From 08c1eebc0c08bb1bea5e0e355f81629cb2ccba1c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 16 Mar 2026 15:34:40 +0000 Subject: [PATCH 2/2] fix(cli): generate .d.ts declaration file for the public API entry point Enable tsup dts generation for src/exports.ts so that dist/exports.d.ts is produced at build time. The binary entry (index) still skips dts. Work around the `incremental: true` in the base tsconfig that conflicts with tsup's DTS worker by overriding it to false in the tsup config. https://claude.ai/code/session_01SExqFNVoBLQCCAbgAY6LXo --- packages/cli/tsup.config.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts index 9b24fe3..688d532 100644 --- a/packages/cli/tsup.config.ts +++ b/packages/cli/tsup.config.ts @@ -6,7 +6,16 @@ export default defineConfig({ exports: "src/exports.ts", }, format: ["esm"], - dts: false, + dts: { + // Only emit declaration files for the public API entry point. + // The `index` (binary) entry doesn't need them. + entry: { exports: "src/exports.ts" }, + compilerOptions: { + // `incremental` is incompatible with tsup's DTS worker when not emitting + // to a single file; disable it here. + incremental: false, + }, + }, clean: true, bundle: true, // External: CommonJS packages that use Node.js built-ins via require()