diff --git a/.knowledge/config.yaml b/.knowledge/config.yaml index 2746a44..4f0e451 100644 --- a/.knowledge/config.yaml +++ b/.knowledge/config.yaml @@ -60,17 +60,4 @@ docsets: - template/index.md - template/markdown-examples.md template: | - You are searching for documentation about '{{keywords}}' in the {{docset_name}} docset. - - **Search Location**: {{local_path}} - **Primary Terms**: {{keywords}} - **Related Context**: {{generalized_keywords}} - - **Search Strategy**: - 1. Start by searching for the primary terms: {{keywords}} - 2. Use tools like `grep`, `rg`, or `find` to locate relevant files - 3. If initial search is too narrow, expand to related terms: {{generalized_keywords}} - 4. Look for file types: .md, .mdx, .vue, .js, .ts files - 5. Check common documentation patterns: README files, guides/, examples/, api/ - - Focus on finding the most relevant documentation that matches your search intent. + Use text search tools (grep, rg, ripgrep) to search for {{keywords}} in {{local_path}}. Try broader terms if needed. Skip: node_modules/, .git/, build/, dist/. diff --git a/.vibe/docs/architecture.md b/.vibe/docs/architecture.md index 58754d5..183e530 100644 --- a/.vibe/docs/architecture.md +++ b/.vibe/docs/architecture.md @@ -162,11 +162,11 @@ Clean separation between protocol handling, business logic, and configuration en **Contained Building Blocks** -| Name | Responsibility | -| -------------------------- | ------------------------------------------------------------------------------- | -| **MCP Server Package** | Handles MCP protocol, tool registration, request routing | -| **Core Package** | Configuration management (ConfigManager), path calculation, template processing | -| **Content Loader Package** | Web source loading, smart content filtering, Git operations | +| Name | Responsibility | +| -------------------------- | -------------------------------------------------------------------------------- | +| **MCP Server Package** | Handles MCP protocol, tool registration, request routing | +| **Core Package** | Configuration management (ConfigManager), path calculation, template processing | +| **Content Loader Package** | Web source loading, smart content filtering, Git operations | | **CLI Package** | Entry point router, user commands for docset management, orchestrates operations | ## Level 2 - Core Package Detail @@ -215,7 +215,7 @@ generateInstructions(template: string, params: TemplateParams): string ```typescript // MCP Tools -search_docs(params: SearchDocsParams): SearchDocsResponse +search_docs(params: SearchDocsParams): { structuredContent: SearchDocsResponse } list_docsets(): ListDocsetsResponse // Server lifecycle @@ -257,7 +257,7 @@ sequenceDiagram Template->>Template: Substitute variables:
{keywords} → "useState, hook"
{generalized_keywords} → "state"
{local_path} → ".knowledge/docs/react-18.2/" Template-->>MCP: "Search for 'useState, hook' in folder .knowledge/docs/react-18.2/..." - MCP-->>AI: SearchDocsResponse with instructions + MCP-->>AI: Structured response with instructions, search_terms, generalized_search_terms, and path Note over AI: Agent uses its own tools (grep, ripgrep, etc.) AI->>FS: Use text search tools on .knowledge/docs/react-18.2/ @@ -279,10 +279,11 @@ sequenceDiagram 4. **Template engine generates instructions** - Substitutes variables in template string - - Returns: "Search for 'useState', 'hook' in folder .knowledge/docs/react-18.2/..." + - Creates structured response with instructions, search_terms, generalized_search_terms, and path -5. **Response sent to AI Assistant** - - Assistant uses its text search tools on the specified path +5. **Structured response sent to AI Assistant** + - Response contains: instructions, search_terms ("useState, hook"), generalized_search_terms ("state"), path (".knowledge/docs/react-18.2/") + - Assistant uses the path and search terms with its text search tools ## Configuration Discovery Scenario diff --git a/packages/core/src/__tests__/template-processor.test.ts b/packages/core/src/__tests__/template-processor.test.ts index 7896a69..fdaa856 100644 --- a/packages/core/src/__tests__/template-processor.test.ts +++ b/packages/core/src/__tests__/template-processor.test.ts @@ -231,7 +231,7 @@ describe("Template Processing", () => { expect(variables).toContain("keywords"); expect(variables).toContain("local_path"); - expect(variables).toContain("generalized_keywords"); + // Note: generalized_keywords no longer required in template since it's provided in structured response }); test("should ensure all variables in default template are allowed", () => { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8c9f1b0..570910a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -32,4 +32,5 @@ export { validateTemplate, extractVariables, createTemplateContext, + createStructuredResponse, } from "./templates/processor.js"; diff --git a/packages/core/src/templates/processor.ts b/packages/core/src/templates/processor.ts index 31e7f09..22d117e 100644 --- a/packages/core/src/templates/processor.ts +++ b/packages/core/src/templates/processor.ts @@ -161,3 +161,25 @@ export function createTemplateContext( docset, }; } + +/** + * Create structured search response + * @param instructions - Processed instruction text + * @param keywords - Search keywords + * @param generalizedKeywords - Generalized keywords + * @param localPath - Calculated local path + * @returns Structured response object + */ +export function createStructuredResponse( + instructions: string, + keywords: string, + generalizedKeywords: string, + localPath: string, +): import("../types.js").SearchDocsResponse { + return { + instructions, + search_terms: keywords, + generalized_search_terms: generalizedKeywords, + path: localPath, + }; +} diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index cd217bd..edce012 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -85,14 +85,12 @@ export interface SearchDocsParams { export interface SearchDocsResponse { /** Instructions for the agent on how to search */ instructions: string; - /** The docset that was searched */ - docset: string; + /** The processed keywords for searching */ + search_terms: string; + /** The processed generalized keywords for broader context */ + generalized_search_terms: string; /** The calculated local path for searching */ - local_path: string; - /** Keywords that were processed */ - keywords: string; - /** Generalized keywords that were processed */ - generalized_keywords: string; + path: string; } /** @@ -148,31 +146,7 @@ export class KnowledgeError extends Error { /** * Default instruction template */ -export const DEFAULT_TEMPLATE = `# 📚 Search {{docset_name}} Documentation - -**Primary terms:** {{keywords}} -**Related terms:** {{generalized_keywords}} -**Location:** {{local_path}} - -## 🔍 Search Strategy - -### 1. **Start with Specific Terms** -Use your text search tools (grep, rg, ripgrep) to search for: \`{{keywords}}\` - -### 2. **Expand to Related Terms** -If initial search doesn't yield results, try: \`{{generalized_keywords}}\` - -### 3. **What to Avoid** -Skip these directories to save time: -- \`node_modules/\`, \`.git/\`, \`.knowledge/\` -- \`build/\`, \`dist/\`, \`target/\`, \`vendor/\` - -## 💡 Search Tips -- Use **case-insensitive** search when possible -- Look for **exact matches first**, then partial matches -- Check **cross-references** and \`See also\` sections -- If stuck, try **broader terms** or ask the user to clarify -`; +export const DEFAULT_TEMPLATE = `Use text search tools (grep, rg, ripgrep) to search for {{keywords}} in {{local_path}}. Try broader terms if needed. Skip: node_modules/, .git/, build/, dist/.`; /** * Allowed template variables that can be used in instruction templates diff --git a/packages/mcp-server/src/__tests__/web-sources.test.ts b/packages/mcp-server/src/__tests__/web-sources.test.ts index 1779420..57feeee 100644 --- a/packages/mcp-server/src/__tests__/web-sources.test.ts +++ b/packages/mcp-server/src/__tests__/web-sources.test.ts @@ -91,17 +91,17 @@ template: "Search for '{{keywords}}' in {{local_path}}. Also consider: {{general const result = await callHandler(request); - expect(result.content).toBeDefined(); - expect(Array.isArray(result.content)).toBe(true); - expect(result.content[0]?.type).toBe("text"); + expect(result.structuredContent).toBeDefined(); + expect(typeof result.structuredContent).toBe("object"); - const instructions = result.content[0]!.text; - expect(instructions).toContain("API documentation"); - expect(instructions).toContain("endpoints methods"); + const response = result.structuredContent; + expect(response.instructions).toContain("API documentation"); + expect(response.search_terms).toContain("API documentation"); + expect(response.generalized_search_terms).toContain("endpoints methods"); // Most importantly: should use standardized path for web sources - expect(instructions).toContain("docsets/web-source-docs"); - expect(instructions).not.toContain("./docs/"); // Should not use local path pattern + expect(response.path).toContain("docsets/web-source-docs"); + expect(response.path).not.toContain("./docs/"); // Should not use local path pattern }); it("WHEN MCP server searches local docset THEN should return configured local_path", async () => { @@ -121,14 +121,15 @@ template: "Search for '{{keywords}}' in {{local_path}}. Also consider: {{general const callHandler = handlers.get("tools/call"); const result = await callHandler(request); - expect(result.content).toBeDefined(); - const instructions = result.content[0]!.text; - expect(instructions).toContain("configuration setup"); - expect(instructions).toContain("install guide"); + expect(result.structuredContent).toBeDefined(); + const response = result.structuredContent; + expect(response.instructions).toContain("configuration setup"); + expect(response.search_terms).toContain("configuration setup"); + expect(response.generalized_search_terms).toContain("install guide"); // Should use the configured local_path for traditional docsets - expect(instructions).toContain("docs/local"); - expect(instructions).not.toContain("docsets/local-docs"); // Should not use docsets pattern + expect(response.path).toContain("docs/local"); + expect(response.path).not.toContain("docsets/local-docs"); // Should not use docsets pattern }); it("WHEN MCP server lists docsets THEN should include both web and local docsets with correct paths", async () => { @@ -225,16 +226,23 @@ template: "Search for '{{keywords}}' in {{local_path}}. Also consider: {{general const callHandler = handlers.get("tools/call"); const result = await callHandler(request); - const instructions = result.content[0]!.text; + const response = result.structuredContent; + + // Should have structured response with correct values + expect(response.search_terms).toContain("authentication middleware"); + expect(response.generalized_search_terms).toContain( + "auth login security", + ); + expect(response.path).toContain("docsets/web-source-docs"); - // Should substitute all template variables correctly - expect(instructions).toContain("authentication middleware"); // {{keywords}} - expect(instructions).toContain("auth login security"); // {{generalized_keywords}} - expect(instructions).toContain("docsets/web-source-docs"); // {{local_path}} for web sources + // Instructions should contain template substitutions + expect(response.instructions).toContain("authentication middleware"); + expect(response.instructions).toContain("auth login security"); + expect(response.instructions).toContain("docsets/web-source-docs"); // Should not contain any unsubstituted template variables - expect(instructions).not.toContain("{{"); - expect(instructions).not.toContain("}}"); + expect(response.instructions).not.toContain("{{"); + expect(response.instructions).not.toContain("}}"); }); }); }); diff --git a/packages/mcp-server/src/server.ts b/packages/mcp-server/src/server.ts index bfe1f34..83cf08b 100644 --- a/packages/mcp-server/src/server.ts +++ b/packages/mcp-server/src/server.ts @@ -15,6 +15,7 @@ import { processTemplate, createTemplateContext, getEffectiveTemplate, + createStructuredResponse, type KnowledgeConfig, } from "@codemcp/knowledge-core"; @@ -93,14 +94,19 @@ export function createAgenticKnowledgeServer() { }) .join("\n"); - const searchDocsDescription = `Search for documentation in available docsets. Returns structured search strategy. + const searchDocsDescription = `Search for documentation in available docsets. Returns structured response with search instructions and parameters. 📚 **AVAILABLE DOCSETS:** ${docsetInfo} -🔍 **SEARCH STRATEGY:** -- Use the tools you have to search in text files (grep, rg, ripgrep, find) -- Start with specific terms, expand to generalized terms`; +🔍 **STRUCTURED RESPONSE:** +Returns JSON object with: +- instructions: Search guidance text +- search_terms: Primary keywords to search for +- generalized_search_terms: Broader terms for context +- path: Local directory path to search in + +Use the path and search terms with your text search tools (grep, rg, ripgrep, find).`; return { tools: [ @@ -241,13 +247,16 @@ ${docsetInfo} templateContext, ); + // Create structured response + const structuredResponse = createStructuredResponse( + instructions, + keywords.trim(), + (generalized_keywords || "").trim(), + localPath, + ); + return { - content: [ - { - type: "text", - text: instructions, - }, - ], + structuredContent: structuredResponse, }; } diff --git a/test/e2e/mcp-protocol-compliance.test.ts b/test/e2e/mcp-protocol-compliance.test.ts index c2c63b3..fe93393 100644 --- a/test/e2e/mcp-protocol-compliance.test.ts +++ b/test/e2e/mcp-protocol-compliance.test.ts @@ -180,18 +180,17 @@ describe("MCP Protocol Compliance E2E Tests", () => { }); expect(result).toBeDefined(); - expect(result.content).toBeDefined(); - expect(Array.isArray(result.content)).toBe(true); - const content = result.content as Array<{ type: string; text: string }>; - expect(content).toHaveLength(1); - expect(content[0]?.type).toBe("text"); + expect(result.structuredContent).toBeDefined(); + expect(typeof result.structuredContent).toBe("object"); - const responseText = content[0]?.text; - expect(responseText).toContain("authentication middleware"); - expect(responseText).toContain("login signin oauth credentials"); - expect(responseText).toContain("Test Documentation"); - expect(responseText).toContain("docs"); + const response = result.structuredContent as any; + expect(response.instructions).toContain("authentication middleware"); + expect(response.search_terms).toContain("authentication middleware"); + expect(response.generalized_search_terms).toContain( + "login signin oauth credentials", + ); + expect(response.path).toContain("docs"); }); it("should execute search_docs with minimal parameters", async () => { @@ -204,15 +203,12 @@ describe("MCP Protocol Compliance E2E Tests", () => { }); expect(result).toBeDefined(); - expect(result.content).toBeDefined(); + expect(result.structuredContent).toBeDefined(); - const content = result.content as Array<{ type: string; text: string }>; - expect(content[0]?.type).toBe("text"); - - const responseText = content[0]?.text; - expect(responseText).toContain("rate limiting"); - expect(responseText).toContain("API Documentation"); - expect(responseText).toContain("api"); + const response = result.structuredContent as any; + expect(response.instructions).toContain("rate limiting"); + expect(response.search_terms).toContain("rate limiting"); + expect(response.path).toContain("api"); }); }); @@ -291,12 +287,12 @@ describe("MCP Protocol Compliance E2E Tests", () => { }, }); - const content = result.content as Array<{ type: string; text: string }>; - const responseText = content[0]?.text; - expect(responseText).toContain("Looking for React information"); - expect(responseText).toContain("useState hook"); - expect(responseText).toContain("state management react hooks"); - expect(responseText).toContain("component names, props, hooks"); + const response = result.structuredContent as any; + expect(response.instructions).toContain("Looking for React information"); + expect(response.search_terms).toContain("useState hook"); + expect(response.generalized_search_terms).toContain( + "state management react hooks", + ); }); });