Skip to content

feat(mcp-server): add init_docset MCP tool #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mrsimpson merged 3 commits into from
Mar 10, 2026
Merged

feat(mcp-server): add init_docset MCP tool #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b5ca44e
feat(mcp-server): add init_docset MCP tool
claude Mar 10, 2026
72e4b77
fix(core): link source contents directly into docset dir (no extra su…
claude Mar 10, 2026
8f8acd5
fix: remove instruction on how to handle un-initialized docsets
mrsimpson Mar 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
319 changes: 23 additions & 296 deletions packages/cli/src/commands/init.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,14 @@

import { Command } from "commander";
import chalk from "chalk";
import { promises as fs } from "node:fs";
import * as path from "node:path";
import {
ConfigManager,
calculateLocalPath,
ensureKnowledgeGitignoreSync,
discoverDirectoryPatterns,
safelyClearDirectory,
getDirectoryInfo,
} from "@codemcp/knowledge-core";
import {
GitRepoLoader,
ArchiveLoader,
WebSourceType,
initDocset,
type SourceResult,
} from "@codemcp/knowledge-content-loader";

export const initCommand = new Command("init")
Expand All @@ -38,13 +32,11 @@ export const initCommand = new Command("init")
console.log(chalk.blue("🚀 Agentic Knowledge Integration Test"));

try {
// Use ConfigManager for all config operations
const configManager = new ConfigManager();
const { config, configPath } = await configManager.loadConfig(
process.cwd(),
);

// Ensure .knowledge/.gitignore exists and contains docsets/ ignore rule
ensureKnowledgeGitignoreSync(configPath);

const docset = config.docsets.find((d) => d.id === docsetId);
Expand All @@ -55,313 +47,48 @@ export const initCommand = new Command("init")
);
}

if (!docset.sources || docset.sources.length === 0) {
throw new Error(`Docset '${docsetId}' has no sources configured`);
}

console.log(chalk.green(`✅ Found docset: ${docset.name}`));
console.log(chalk.gray(`📝 Description: ${docset.description}`));
console.log(chalk.gray(`🔗 Sources: ${docset.sources.length}`));

// Calculate the local path for this docset
const localPath = calculateLocalPath(docset, configPath);

console.log(chalk.yellow(`\n📁 Target directory: ${localPath}`));

// Check if already exists
let existsAlready = false;
try {
const stat = await fs.stat(localPath);
if (stat.isDirectory()) {
existsAlready = true;
}
} catch {
// Directory doesn't exist, which is fine
}

if (existsAlready && !options.force) {
console.log(
chalk.yellow(
"⚠️ Directory already exists. Use --force to overwrite.",
),
);
const files = await fs.readdir(localPath);
console.log(
chalk.gray(
`Existing files: ${files.slice(0, 5).join(", ")}${files.length > 5 ? "..." : ""}`,
),
);
return;
}

// Clear directory for force re-initialization
if (existsAlready && options.force) {
// Get info about what we're clearing (for logging)
const dirInfo = await getDirectoryInfo(localPath);

console.log(chalk.yellow("🗑️ Clearing existing directory..."));
console.log(
chalk.gray(
` Removing: ${dirInfo.files} files, ${dirInfo.directories} dirs, ${dirInfo.symlinks} symlinks`,
),
);

if (dirInfo.symlinks > 0) {
const result = await initDocset(docsetId, docset, configPath, {
force: options.force,
onSourceProgress: (sourceResult: SourceResult) => {
const icon =
sourceResult.type === "git_repo"
? "Copied"
: sourceResult.type === "local_folder"
? "Created symlinks:"
: "Extracted";
console.log(
chalk.gray(
" ⚠️ Note: Symlinks will be removed, but source files are preserved",
),
chalk.green(` ✅ ${icon} — ${sourceResult.message}`),
);
}

// Safely clear directory (preserves source files for symlinked folders)
await safelyClearDirectory(localPath);
}
},
});

// Create target directory
await fs.mkdir(localPath, { recursive: true });

let totalFiles = 0;
const allDiscoveredPaths: string[] = [];

// Process each source
for (const [index, source] of docset.sources.entries()) {
if (result.alreadyInitialized) {
console.log(
chalk.yellow(
`\n🔄 Loading source ${index + 1}/${docset.sources.length}: ${source.type === "git_repo" ? source.url : source.paths?.join(", ")}`,
"⚠️ Directory already exists and is initialized. Use --force to overwrite.",
),
);

if (source.type === "git_repo") {
// Use GitRepoLoader for all Git operations (REQ-19)
const loader = new GitRepoLoader();

console.log(
chalk.gray(` Using GitRepoLoader for smart content filtering`),
);

const webSourceConfig = {
url: source.url,
type: WebSourceType.GIT_REPO,
options: {
branch: source.branch || "main",
paths: source.paths || [],
},
};

// Validate configuration
const validation = loader.validateConfig(webSourceConfig);
if (validation !== true) {
throw new Error(
`Invalid Git repository configuration: ${validation}`,
);
}

// Load content using GitRepoLoader
const result = await loader.load(webSourceConfig, localPath);

if (!result.success) {
throw new Error(`Git repository loading failed: ${result.error}`);
}

// Collect discovered paths for config update
allDiscoveredPaths.push(...result.files);

totalFiles += result.files.length;
console.log(
chalk.green(
` ✅ Copied ${result.files.length} files using smart filtering`,
),
);

// Create source metadata
const metadata = {
source_url: source.url,
source_type: source.type,
downloaded_at: new Date().toISOString(),
files_count: result.files.length,
files: result.files,
docset_id: docsetId,
content_hash: result.contentHash,
};

await fs.writeFile(
path.join(localPath, `.agentic-source-${index}.json`),
JSON.stringify(metadata, null, 2),
);
} else if (source.type === "local_folder") {
// Handle local folder initialization
console.log(chalk.gray(` Creating symlinks for local folder`));

if (!source.paths || source.paths.length === 0) {
throw new Error(`Local folder source has no paths configured`);
}

// Import symlink utilities
const { createSymlinks } = await import("@codemcp/knowledge-core");

// Note: directory is already cleared above if --force is used,
// so no need to call removeSymlinks here

const configDir = path.dirname(configPath);
const projectRoot = path.dirname(configDir);

// Verify source paths exist
const validatedPaths: string[] = [];
for (const sourcePath of source.paths) {
const absolutePath = path.isAbsolute(sourcePath)
? sourcePath
: path.resolve(projectRoot, sourcePath);

try {
const stat = await fs.stat(absolutePath);
if (!stat.isDirectory()) {
throw new Error(`Path is not a directory: ${sourcePath}`);
}
validatedPaths.push(sourcePath);
} catch {
throw new Error(
`Local folder path does not exist: ${sourcePath}`,
);
}
}

// Create symlinks
await createSymlinks(validatedPaths, localPath, projectRoot);

console.log(
chalk.green(` ✅ Created ${validatedPaths.length} symlink(s)`),
);

// Count files in symlinked directories for metadata
let fileCount = 0;
const files: string[] = [];

async function countFilesRecursive(dir: string): Promise<void> {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
await countFilesRecursive(fullPath);
} else if (entry.isFile()) {
fileCount++;
files.push(path.relative(localPath, fullPath));
}
}
}

await countFilesRecursive(localPath);
totalFiles += fileCount;

// Create source metadata
const metadata = {
source_paths: validatedPaths,
source_type: source.type,
initialized_at: new Date().toISOString(),
files_count: fileCount,
files: files,
docset_id: docsetId,
};

await fs.writeFile(
path.join(localPath, `.agentic-source-${index}.json`),
JSON.stringify(metadata, null, 2),
);
} else if (source.type === "archive") {
// Handle archive file initialization (zip, tar.gz, etc.)
const loader = new ArchiveLoader();
const sourceUrl = source.url || source.path || "";

console.log(
chalk.gray(` Using ArchiveLoader for archive extraction`),
);

const webSourceConfig = {
url: sourceUrl,
type: WebSourceType.ARCHIVE,
options: {
paths: source.paths || [],
},
};

// Validate configuration
const validation = loader.validateConfig(webSourceConfig);
if (validation !== true) {
throw new Error(
`Invalid archive source configuration: ${validation}`,
);
}

// Load content using ArchiveLoader
const result = await loader.load(webSourceConfig, localPath);

if (!result.success) {
throw new Error(`Archive loading failed: ${result.error}`);
}

// Collect discovered paths for config update
allDiscoveredPaths.push(...result.files);

totalFiles += result.files.length;
console.log(
chalk.green(
` ✅ Extracted ${result.files.length} files from archive`,
),
);

// Create source metadata
const metadata = {
source_url: sourceUrl,
source_type: source.type,
downloaded_at: new Date().toISOString(),
files_count: result.files.length,
files: result.files,
docset_id: docsetId,
content_hash: result.contentHash,
};

await fs.writeFile(
path.join(localPath, `.agentic-source-${index}.json`),
JSON.stringify(metadata, null, 2),
);
} else {
console.log(
chalk.red(
` ❌ Source type '${(source as any).type}' not yet supported`,
),
);
}
return;
}

// Create overall metadata
const overallMetadata = {
docset_id: docsetId,
docset_name: docset.name,
initialized_at: new Date().toISOString(),
total_files: totalFiles,
sources_count: docset.sources.length,
};

await fs.writeFile(
path.join(localPath, ".agentic-metadata.json"),
JSON.stringify(overallMetadata, null, 2),
);

// Update configuration with discovered paths (only if --discover-paths flag used)
if (allDiscoveredPaths.length > 0 && options.discoverPaths) {
const allFiles = result.sourceResults.flatMap((r) => r.files);
if (allFiles.length > 0 && options.discoverPaths) {
console.log(
chalk.yellow(
`\n📝 Discovering directory patterns from extracted files...`,
),
);

// Convert file list to directory patterns
const directoryPatterns =
discoverDirectoryPatterns(allDiscoveredPaths);
const directoryPatterns = discoverDirectoryPatterns(allFiles);

console.log(
chalk.gray(
` Found ${allDiscoveredPaths.length} files → ${directoryPatterns.length} patterns`,
` Found ${allFiles.length} files → ${directoryPatterns.length} patterns`,
),
);

Expand All @@ -384,8 +111,8 @@ export const initCommand = new Command("init")
console.log(
chalk.green(`\n🎉 Successfully initialized docset '${docsetId}'`),
);
console.log(chalk.gray(`📁 Location: ${localPath}`));
console.log(chalk.gray(`📄 Total files: ${totalFiles}`));
console.log(chalk.gray(`📁 Location: ${result.localPath}`));
console.log(chalk.gray(`📄 Total files: ${result.totalFiles}`));
console.log(
chalk.gray(`🔗 Sources processed: ${docset.sources.length}`),
);
Expand Down
1 change: 1 addition & 0 deletions packages/content-loader/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@codemcp/knowledge-core": "workspace:*",
"adm-zip": "0.5.16",
"simple-git": "^3.22.0",
"tar": "7.5.9"
Expand Down
Loading
Loading