feat: implement structured JSON response for search_docs tool

.knowledge/config.yaml

@@ -60,17 +60,4 @@ docsets:
           - template/index.md
           - template/markdown-examples.md
 template: |
-  You are searching for documentation about '{{keywords}}' in the {{docset_name}} docset.
-
-  **Search Location**: {{local_path}}
-  **Primary Terms**: {{keywords}}
-  **Related Context**: {{generalized_keywords}}
-
-  **Search Strategy**:
-  1. Start by searching for the primary terms: {{keywords}}
-  2. Use tools like `grep`, `rg`, or `find` to locate relevant files
-  3. If initial search is too narrow, expand to related terms: {{generalized_keywords}}
-  4. Look for file types: .md, .mdx, .vue, .js, .ts files
-  5. Check common documentation patterns: README files, guides/, examples/, api/
-
-  Focus on finding the most relevant documentation that matches your search intent.
+  Use text search tools (grep, rg, ripgrep) to search for {{keywords}} in {{local_path}}. Try broader terms if needed. Skip: node_modules/, .git/, build/, dist/.

.vibe/docs/architecture.md

@@ -162,11 +162,11 @@ Clean separation between protocol handling, business logic, and configuration en
 
 **Contained Building Blocks**
 
-| Name                       | Responsibility                                                                  |
-| -------------------------- | ------------------------------------------------------------------------------- |
-| **MCP Server Package**     | Handles MCP protocol, tool registration, request routing                        |
-| **Core Package**           | Configuration management (ConfigManager), path calculation, template processing |
-| **Content Loader Package** | Web source loading, smart content filtering, Git operations                     |
+| Name                       | Responsibility                                                                   |
+| -------------------------- | -------------------------------------------------------------------------------- |
+| **MCP Server Package**     | Handles MCP protocol, tool registration, request routing                         |
+| **Core Package**           | Configuration management (ConfigManager), path calculation, template processing  |
+| **Content Loader Package** | Web source loading, smart content filtering, Git operations                      |
 | **CLI Package**            | Entry point router, user commands for docset management, orchestrates operations |
 
 ## Level 2 - Core Package Detail
@@ -215,7 +215,7 @@ generateInstructions(template: string, params: TemplateParams): string
 
 ```typescript
 // MCP Tools
-search_docs(params: SearchDocsParams): SearchDocsResponse
+search_docs(params: SearchDocsParams): { structuredContent: SearchDocsResponse }
 list_docsets(): ListDocsetsResponse
 
 // Server lifecycle
@@ -257,7 +257,7 @@ sequenceDiagram
     Template->>Template: Substitute variables:<br/>{keywords} → "useState, hook"<br/>{generalized_keywords} → "state"<br/>{local_path} → ".knowledge/docs/react-18.2/"
     Template-->>MCP: "Search for 'useState, hook' in folder .knowledge/docs/react-18.2/..."
 
-    MCP-->>AI: SearchDocsResponse with instructions
+    MCP-->>AI: Structured response with instructions, search_terms, generalized_search_terms, and path
 
     Note over AI: Agent uses its own tools (grep, ripgrep, etc.)
     AI->>FS: Use text search tools on .knowledge/docs/react-18.2/
@@ -279,10 +279,11 @@ sequenceDiagram
 
 4. **Template engine generates instructions**
    - Substitutes variables in template string
-   - Returns: "Search for 'useState', 'hook' in folder .knowledge/docs/react-18.2/..."
+   - Creates structured response with instructions, search_terms, generalized_search_terms, and path
 
-5. **Response sent to AI Assistant**
-   - Assistant uses its text search tools on the specified path
+5. **Structured response sent to AI Assistant**
+   - Response contains: instructions, search_terms ("useState, hook"), generalized_search_terms ("state"), path (".knowledge/docs/react-18.2/")
+   - Assistant uses the path and search terms with its text search tools
 
 ## Configuration Discovery Scenario
 

packages/core/src/__tests__/template-processor.test.ts

@@ -231,7 +231,7 @@ describe("Template Processing", () => {
 
       expect(variables).toContain("keywords");
       expect(variables).toContain("local_path");
-      expect(variables).toContain("generalized_keywords");
+      // Note: generalized_keywords no longer required in template since it's provided in structured response
     });
 
     test("should ensure all variables in default template are allowed", () => {

packages/core/src/index.ts

@@ -32,4 +32,5 @@ export {
   validateTemplate,
   extractVariables,
   createTemplateContext,
+  createStructuredResponse,
 } from "./templates/processor.js";

packages/core/src/templates/processor.ts

@@ -161,3 +161,25 @@ export function createTemplateContext(
     docset,
   };
 }
+
+/**
+ * Create structured search response
+ * @param instructions - Processed instruction text
+ * @param keywords - Search keywords
+ * @param generalizedKeywords - Generalized keywords
+ * @param localPath - Calculated local path
+ * @returns Structured response object
+ */
+export function createStructuredResponse(
+  instructions: string,
+  keywords: string,
+  generalizedKeywords: string,
+  localPath: string,
+): import("../types.js").SearchDocsResponse {
+  return {
+    instructions,
+    search_terms: keywords,
+    generalized_search_terms: generalizedKeywords,
+    path: localPath,
+  };
+}

packages/core/src/types.ts

@@ -85,14 +85,12 @@
 export interface SearchDocsResponse {
   /** Instructions for the agent on how to search */
   instructions: string;
-  /** The docset that was searched */
-  docset: string;
+  /** The processed keywords for searching */
+  search_terms: string;
+  /** The processed generalized keywords for broader context */
+  generalized_search_terms: string;
   /** The calculated local path for searching */
-  local_path: string;
-  /** Keywords that were processed */
-  keywords: string;
-  /** Generalized keywords that were processed */
-  generalized_keywords: string;
+  path: string;
 }
 
 /**
@@ -123,12 +121,12 @@
  * Error types that can occur in the core system
  */
 export enum ErrorType {
   CONFIG_NOT_FOUND = "CONFIG_NOT_FOUND",
   CONFIG_INVALID = "CONFIG_INVALID",
   DOCSET_NOT_FOUND = "DOCSET_NOT_FOUND",
   PATH_INVALID = "PATH_INVALID",
   TEMPLATE_ERROR = "TEMPLATE_ERROR",
   YAML_PARSE_ERROR = "YAML_PARSE_ERROR",
 }
 
 /**
@@ -148,31 +146,7 @@
 /**
  * Default instruction template
  */
-export const DEFAULT_TEMPLATE = `# 📚 Search {{docset_name}} Documentation
-
-**Primary terms:** {{keywords}}  
-**Related terms:** {{generalized_keywords}}  
-**Location:** {{local_path}}
-
-## 🔍 Search Strategy
-
-### 1. **Start with Specific Terms**
-Use your text search tools (grep, rg, ripgrep) to search for: \`{{keywords}}\`
-
-### 2. **Expand to Related Terms**
-If initial search doesn't yield results, try: \`{{generalized_keywords}}\`
-
-### 3. **What to Avoid**
-Skip these directories to save time:
-- \`node_modules/\`, \`.git/\`, \`.knowledge/\`
-- \`build/\`, \`dist/\`, \`target/\`, \`vendor/\`
-
-## 💡 Search Tips
-- Use **case-insensitive** search when possible
-- Look for **exact matches first**, then partial matches
-- Check **cross-references** and \`See also\` sections
-- If stuck, try **broader terms** or ask the user to clarify
-`;
+export const DEFAULT_TEMPLATE = `Use text search tools (grep, rg, ripgrep) to search for {{keywords}} in {{local_path}}. Try broader terms if needed. Skip: node_modules/, .git/, build/, dist/.`;
 
 /**
  * Allowed template variables that can be used in instruction templates

packages/mcp-server/src/__tests__/web-sources.test.ts

@@ -91,17 +91,17 @@ template: "Search for '{{keywords}}' in {{local_path}}. Also consider: {{general
 
       const result = await callHandler(request);
 
-      expect(result.content).toBeDefined();
-      expect(Array.isArray(result.content)).toBe(true);
-      expect(result.content[0]?.type).toBe("text");
+      expect(result.structuredContent).toBeDefined();
+      expect(typeof result.structuredContent).toBe("object");
 
-      const instructions = result.content[0]!.text;
-      expect(instructions).toContain("API documentation");
-      expect(instructions).toContain("endpoints methods");
+      const response = result.structuredContent;
+      expect(response.instructions).toContain("API documentation");
+      expect(response.search_terms).toContain("API documentation");
+      expect(response.generalized_search_terms).toContain("endpoints methods");
 
       // Most importantly: should use standardized path for web sources
-      expect(instructions).toContain("docsets/web-source-docs");
-      expect(instructions).not.toContain("./docs/"); // Should not use local path pattern
+      expect(response.path).toContain("docsets/web-source-docs");
+      expect(response.path).not.toContain("./docs/"); // Should not use local path pattern
     });
 
     it("WHEN MCP server searches local docset THEN should return configured local_path", async () => {
@@ -121,14 +121,15 @@ template: "Search for '{{keywords}}' in {{local_path}}. Also consider: {{general
       const callHandler = handlers.get("tools/call");
       const result = await callHandler(request);
 
-      expect(result.content).toBeDefined();
-      const instructions = result.content[0]!.text;
-      expect(instructions).toContain("configuration setup");
-      expect(instructions).toContain("install guide");
+      expect(result.structuredContent).toBeDefined();
+      const response = result.structuredContent;
+      expect(response.instructions).toContain("configuration setup");
+      expect(response.search_terms).toContain("configuration setup");
+      expect(response.generalized_search_terms).toContain("install guide");
 
       // Should use the configured local_path for traditional docsets
-      expect(instructions).toContain("docs/local");
-      expect(instructions).not.toContain("docsets/local-docs"); // Should not use docsets pattern
+      expect(response.path).toContain("docs/local");
+      expect(response.path).not.toContain("docsets/local-docs"); // Should not use docsets pattern
     });
 
     it("WHEN MCP server lists docsets THEN should include both web and local docsets with correct paths", async () => {
@@ -225,16 +226,23 @@ template: "Search for '{{keywords}}' in {{local_path}}. Also consider: {{general
       const callHandler = handlers.get("tools/call");
       const result = await callHandler(request);
 
-      const instructions = result.content[0]!.text;
+      const response = result.structuredContent;
+
+      // Should have structured response with correct values
+      expect(response.search_terms).toContain("authentication middleware");
+      expect(response.generalized_search_terms).toContain(
+        "auth login security",
+      );
+      expect(response.path).toContain("docsets/web-source-docs");
 
-      // Should substitute all template variables correctly
-      expect(instructions).toContain("authentication middleware"); // {{keywords}}
-      expect(instructions).toContain("auth login security"); // {{generalized_keywords}}
-      expect(instructions).toContain("docsets/web-source-docs"); // {{local_path}} for web sources
+      // Instructions should contain template substitutions
+      expect(response.instructions).toContain("authentication middleware");
+      expect(response.instructions).toContain("auth login security");
+      expect(response.instructions).toContain("docsets/web-source-docs");
 
       // Should not contain any unsubstituted template variables
-      expect(instructions).not.toContain("{{");
-      expect(instructions).not.toContain("}}");
+      expect(response.instructions).not.toContain("{{");
+      expect(response.instructions).not.toContain("}}");
     });
   });
 });

packages/mcp-server/src/server.ts

@@ -15,6 +15,7 @@
   processTemplate,
   createTemplateContext,
   getEffectiveTemplate,
+  createStructuredResponse,
   type KnowledgeConfig,
 } from "@codemcp/knowledge-core";
 
@@ -93,14 +94,19 @@
         })
         .join("\n");
 
-      const searchDocsDescription = `Search for documentation in available docsets. Returns structured search strategy.
+      const searchDocsDescription = `Search for documentation in available docsets. Returns structured response with search instructions and parameters.
 
 📚 **AVAILABLE DOCSETS:**
 ${docsetInfo}
 
-🔍 **SEARCH STRATEGY:**
-- Use the tools you have to search in text files (grep, rg, ripgrep, find)
-- Start with specific terms, expand to generalized terms`;
+🔍 **STRUCTURED RESPONSE:**
+Returns JSON object with:
+- instructions: Search guidance text
+- search_terms: Primary keywords to search for
+- generalized_search_terms: Broader terms for context
+- path: Local directory path to search in
+
+Use the path and search terms with your text search tools (grep, rg, ripgrep, find).`;
 
       return {
         tools: [
@@ -142,7 +148,7 @@
           },
         ],
       };
     } catch (error) {
       // Fallback to basic tools if configuration fails
       return {
         tools: [
@@ -241,13 +247,16 @@
             templateContext,
           );
 
+          // Create structured response
+          const structuredResponse = createStructuredResponse(
+            instructions,
+            keywords.trim(),
+            (generalized_keywords || "").trim(),
+            localPath,
+          );
+
           return {
-            content: [
-              {
-                type: "text",
-                text: instructions,
-              },
-            ],
+            structuredContent: structuredResponse,
           };
         }
 

test/e2e/mcp-protocol-compliance.test.ts

@@ -180,18 +180,17 @@ describe("MCP Protocol Compliance E2E Tests", () => {
       });
 
       expect(result).toBeDefined();
-      expect(result.content).toBeDefined();
-      expect(Array.isArray(result.content)).toBe(true);
 
-      const content = result.content as Array<{ type: string; text: string }>;
-      expect(content).toHaveLength(1);
-      expect(content[0]?.type).toBe("text");
+      expect(result.structuredContent).toBeDefined();
+      expect(typeof result.structuredContent).toBe("object");
 
-      const responseText = content[0]?.text;
-      expect(responseText).toContain("authentication middleware");
-      expect(responseText).toContain("login signin oauth credentials");
-      expect(responseText).toContain("Test Documentation");
-      expect(responseText).toContain("docs");
+      const response = result.structuredContent as any;
+      expect(response.instructions).toContain("authentication middleware");
+      expect(response.search_terms).toContain("authentication middleware");
+      expect(response.generalized_search_terms).toContain(
+        "login signin oauth credentials",
+      );
+      expect(response.path).toContain("docs");
     });
 
     it("should execute search_docs with minimal parameters", async () => {
@@ -204,15 +203,12 @@ describe("MCP Protocol Compliance E2E Tests", () => {
       });
 
       expect(result).toBeDefined();
-      expect(result.content).toBeDefined();
+      expect(result.structuredContent).toBeDefined();
 
-      const content = result.content as Array<{ type: string; text: string }>;
-      expect(content[0]?.type).toBe("text");
-
-      const responseText = content[0]?.text;
-      expect(responseText).toContain("rate limiting");
-      expect(responseText).toContain("API Documentation");
-      expect(responseText).toContain("api");
+      const response = result.structuredContent as any;
+      expect(response.instructions).toContain("rate limiting");
+      expect(response.search_terms).toContain("rate limiting");
+      expect(response.path).toContain("api");
     });
   });
 
@@ -291,12 +287,12 @@ describe("MCP Protocol Compliance E2E Tests", () => {
         },
       });
 
-      const content = result.content as Array<{ type: string; text: string }>;
-      const responseText = content[0]?.text;
-      expect(responseText).toContain("Looking for React information");
-      expect(responseText).toContain("useState hook");
-      expect(responseText).toContain("state management react hooks");
-      expect(responseText).toContain("component names, props, hooks");
+      const response = result.structuredContent as any;
+      expect(response.instructions).toContain("Looking for React information");
+      expect(response.search_terms).toContain("useState hook");
+      expect(response.generalized_search_terms).toContain(
+        "state management react hooks",
+      );
     });
   });