diff --git a/Dockerfile b/Dockerfile index 82313fd..be902a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,9 @@ RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci --omit=dev +# Optional: install these if your config uses document sources or local embeddings +# RUN npm install pdf-parse mammoth # For type: document (PDF/DOCX) +# RUN npm install @xenova/transformers # For embedding.provider: local COPY --from=build /app/dist/ ./dist/ COPY deploy/copilotkit-docs.yaml ./copilotkit-docs.yaml COPY deploy/pathfinder-docs.yaml ./pathfinder-docs.yaml diff --git a/docs/deploy/index.html b/docs/deploy/index.html index 34f95cd..3eca648 100644 --- a/docs/deploy/index.html +++ b/docs/deploy/index.html @@ -416,7 +416,22 @@

Environment Variables

Optional dependencies

-

For document source type (PDF/DOCX), install optional peer dependencies: npm install pdf-parse mammoth

+

Some features require extra packages that are not bundled by default. Install only the ones your config needs:

+ + + + + + + + + + + + + +
PackageWhen requiredInstall
pdf-parseAny source has type: document with *.pdf file patternsnpm install pdf-parse
mammothAny source has type: document with *.docx file patternsnpm install mammoth
@xenova/transformersembedding.provider is localnpm install @xenova/transformers
+

Run pathfinder validate to detect missing optional dependencies and get install instructions.

Health endpoint

Pathfinder exposes GET /health for monitoring. It returns JSON with uptime, indexing status, chunk counts per source, and index state (last indexed time, commit SHA, errors). Use it for load balancer health checks and deployment verification.

diff --git a/docs/index.html b/docs/index.html index 7e21f33..cfd3722 100644 --- a/docs/index.html +++ b/docs/index.html @@ -37,7 +37,15 @@ --warning: #ffaa00; --mono: "JetBrains Mono", "SF Mono", "Fira Code", monospace; --sans: "Instrument Sans", -apple-system, system-ui, sans-serif; + --ease-out-expo: cubic-bezier(0.16, 1, 0.3, 1); } +/* Animations */ +@keyframes fadeUp { + from { opacity: 0; transform: translateY(30px); } + to { opacity: 1; transform: translateY(0); } +} +.fade-in { opacity: 0; transform: translateY(30px); transition: opacity 0.8s var(--ease-out-expo), transform 0.8s var(--ease-out-expo); } +.fade-in.visible { opacity: 1; transform: translateY(0); } * { margin:0; padding:0; box-sizing:border-box; } html { -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; font-size: 16px; } body { background: var(--bg-deep); color: var(--text-primary); font-family: var(--sans); line-height: 1.6; overflow-x:hidden; } @@ -66,14 +74,14 @@ /* Hero */ .hero { padding: 120px 0 3rem; max-width: 1120px; margin: 0 auto; padding-left: 2rem; padding-right: 2rem; } -.badge { display: inline-flex; align-items:center; gap: 8px; padding: 6px 16px; border: 1px solid var(--border); border-radius: 100px; font-family: var(--mono); font-size: 0.75rem; color: var(--text-secondary); margin-bottom: 32px; } +.badge { display: inline-flex; align-items:center; gap: 8px; padding: 6px 16px; border: 1px solid var(--border); border-radius: 100px; font-family: var(--mono); font-size: 0.75rem; color: var(--text-secondary); margin-bottom: 32px; animation: fadeUp 0.8s var(--ease-out-expo) both; } .badge .dot { width: 8px; height: 8px; border-radius: 50%; background: var(--accent); animation: pulse 2s infinite; } @keyframes pulse { 0%,100% { opacity:1; } 50% { opacity:0.4; } } -.hero h1 { font-size: clamp(2.5rem, 6vw, 4.5rem); font-weight: 700; line-height: 1.08; margin-bottom: 24px; letter-spacing: -0.03em; } +.hero h1 { font-size: clamp(2.5rem, 6vw, 4.5rem); font-weight: 700; line-height: 1.08; margin-bottom: 24px; letter-spacing: -0.03em; animation: fadeUp 0.8s var(--ease-out-expo) 0.1s both; } .hero-break { display: none; } @media (min-width: 769px) { .hero-break { display: block; } } .hero h1 .accent { color: var(--accent); } -.hero .subtitle { font-size: 1.2rem; color: var(--text-secondary); max-width: 700px; line-height: 1.7; margin-bottom: 40px; } +.hero .subtitle { font-size: 1.2rem; color: var(--text-secondary); max-width: 700px; line-height: 1.7; margin-bottom: 40px; animation: fadeUp 0.8s var(--ease-out-expo) 0.2s both; } /* Install box */ .install-box { display: inline-flex; flex-direction: column; gap: 0; background: var(--bg-surface); border: 1px solid var(--border); border-radius: 8px; padding: 12px 20px; font-family: var(--mono); font-size: 0.9rem; margin-bottom: 2.5rem; } @@ -104,7 +112,7 @@ /* Feature cards (3x3 grid) */ .feature-grid { display: grid; grid-template-columns: repeat(3, 1fr); gap: 1.5rem; } -.feature-card { background: var(--bg-surface); border: 1px solid var(--border); border-radius: 12px; padding: 2rem; box-shadow: 0 4px 16px rgba(0,0,0,0.2); transition: transform 0.2s, border-color 0.2s, box-shadow 0.2s; } +.feature-card { background: var(--bg-surface); border: 1px solid var(--border); border-radius: 12px; padding: 2rem; box-shadow: 0 4px 16px rgba(0,0,0,0.2); transition: all 0.3s var(--ease-out-expo); } .feature-card:hover { transform: translateY(-4px); border-color: var(--accent-dim); box-shadow: 0 8px 32px rgba(0,0,0,0.4); } .feature-card .icon { font-size: 1.75rem; margin-bottom: 1rem; } .feature-card h3 { font-size: 1.05rem; font-weight: 600; margin-bottom: 0.5rem; } @@ -197,6 +205,9 @@ $ pathfinder
Home + Features + Comparison + Switch Docs GitHub
@@ -210,6 +221,9 @@
Home + Features + Comparison + Switch Docs GitHub
@@ -269,51 +283,51 @@

The knowledge server
for
-

What's in the box

-

A complete system enabling agents to discover and learn what's important.

+

What's in the box

+

A complete system enabling agents to discover and learn what's important.

-
+
🔍

Semantic + Filesystem

Agents choose the right tool: vector search for conceptual questions, grep and cat for precise lookups. Both paradigms, composable.

-
+
💬

Automatic Q&A Extraction

Slack threads and Discord forums are distilled into Q&A pairs automatically. Forum posts need no LLM — they're already questions and answers.

-
+
🔗

Cross-Source Intelligence

Search docs and Q&A together. Grep misses suggest semantic search. related finds similar files across sources.

-
+
📋

Knowledge on Tap

/faq.txt and /llms.txt give agents instant context at session start. No search required — your best answers are already loaded.

-
+
🚀

Zero-Infra Mode

Bash-only tools work with no database, no API keys. Add RAG when you're ready — it's just config.

-
+
🔄

Always Current

Webhook-triggered reindexing on every push. Nightly full reindex on schedule. Confidence thresholds adjust without re-processing.

-
+
🔌

Pluggable Embeddings

OpenAI, Ollama, or local transformers.js — choose your embedding provider. Run fully local with no API keys, or use the cloud. Switch with one config line.

-
+
📄

PDF & DOCX Support

Index binary documents directly — no conversion needed. Page-break and section-aware chunking extracts structure from PDFs and Word docs.

-
+
📊

Analytics Dashboard

See what agents search for, which queries return empty, and where your knowledge gaps are. Built-in dashboard at /analytics.

@@ -322,10 +336,10 @@

Analytics Dashboard

-

How Pathfinder compares

-

See how Pathfinder stacks up against other knowledge tools for AI agents.

+

How Pathfinder compares

+

See how Pathfinder stacks up against other knowledge tools for AI agents.

-
+
@@ -464,9 +478,9 @@

How Pathfinder compares

-

Switch to Pathfinder

-

Migration guides for every platform. Most take less than 15 minutes.

-
+

Switch to Pathfinder

+

Migration guides for every platform. Most take less than 15 minutes.

+
From Mintlify → From GitBook → From mcp-ragdocs → @@ -577,5 +591,25 @@

Switch to Pathfinder

}); })(); + \ No newline at end of file diff --git a/src/__tests__/validate.test.ts b/src/__tests__/validate.test.ts index bc890ac..3b313e4 100644 --- a/src/__tests__/validate.test.ts +++ b/src/__tests__/validate.test.ts @@ -467,6 +467,121 @@ describe("validateConfig", () => { expect(openaiEnv?.required).toBe(true); }); + // ── Optional dependency checks ────────────────────────────────────────── + + describe("optional dependency checks", () => { + it("reports missing pdf-parse for PDF document sources", async () => { + (getServerConfig as ReturnType).mockReturnValue({ + server: { name: "test", version: "1.0" }, + sources: [ + { + name: "pdf-docs", + type: "document", + path: "./docs", + file_patterns: ["**/*.pdf"], + chunk: {}, + }, + ], + tools: [ + { + name: "c", + type: "collect", + description: "collect", + response: "Collected", + schema: { field1: { type: "string" } }, + }, + ], + }); + + const result = await validateConfig(); + expect( + result.errors.some( + (e) => + e.includes("Missing optional dependency: pdf-parse") && + e.includes("npm install pdf-parse"), + ), + ).toBe(true); + }); + + it("reports missing mammoth for DOCX document sources", async () => { + (getServerConfig as ReturnType).mockReturnValue({ + server: { name: "test", version: "1.0" }, + sources: [ + { + name: "docx-docs", + type: "document", + path: "./docs", + file_patterns: ["**/*.docx"], + chunk: {}, + }, + ], + tools: [ + { + name: "c", + type: "collect", + description: "collect", + response: "Collected", + schema: { field1: { type: "string" } }, + }, + ], + }); + + const result = await validateConfig(); + expect( + result.errors.some( + (e) => + e.includes("Missing optional dependency: mammoth") && + e.includes("npm install mammoth"), + ), + ).toBe(true); + }); + + it("reports missing @xenova/transformers for local embedding provider", async () => { + (getServerConfig as ReturnType).mockReturnValue({ + server: { name: "test", version: "1.0" }, + sources: [ + { + name: "docs", + type: "markdown", + path: "./docs", + file_patterns: ["**/*.md"], + chunk: {}, + }, + ], + tools: [ + { + name: "c", + type: "collect", + description: "collect", + response: "Collected", + schema: { field1: { type: "string" } }, + }, + ], + embedding: { + provider: "local", + model: "Xenova/all-MiniLM-L6-v2", + dimensions: 384, + }, + }); + + const result = await validateConfig(); + expect( + result.errors.some( + (e) => + e.includes("Missing optional dependency: @xenova/transformers") && + e.includes("npm install @xenova/transformers"), + ), + ).toBe(true); + }); + + it("does not report optional dep errors when no document sources or local embeddings", async () => { + const result = await validateConfig(); + expect( + result.errors.some((e) => e.includes("Missing optional dependency")), + ).toBe(false); + }); + }); + // ── Source validation ──────────────────────────────────────────────────── describe("source validation", () => { @@ -946,6 +1061,52 @@ describe("formatValidationResult", () => { expect(output).toContain("- Error three"); }); + it("formats optional dependency warnings separately from hard errors", () => { + const result: ValidationResult = { + configValid: true, + envVars: [], + sources: [], + tools: [], + errors: [ + "Missing optional dependency: pdf-parse — Required for PDF document sources. Install: npm install pdf-parse", + "Missing required environment variable: DATABASE_URL", + ], + }; + + const output = formatValidationResult(result); + expect(output).toContain("Optional Dependencies:"); + expect(output).toContain("pdf-parse"); + expect(output).toContain("2 error(s) found"); + expect(output).toContain( + "- Missing required environment variable: DATABASE_URL", + ); + // The optional dep error should appear in the Optional Dependencies section, not in the error list + const lines = output.split("\n"); + const errorListLines = lines.filter( + (l) => l.startsWith(" - ") && l.includes("Missing required"), + ); + expect(errorListLines).toHaveLength(1); + }); + + it("formats result with only optional dep warnings as no hard errors", () => { + const result: ValidationResult = { + configValid: true, + envVars: [], + sources: [], + tools: [], + errors: [ + "Missing optional dependency: mammoth — Required for DOCX document sources. Install: npm install mammoth", + ], + }; + + const output = formatValidationResult(result); + expect(output).toContain("Optional Dependencies:"); + expect(output).toContain( + "1 optional dependency warning(s), no hard errors", + ); + expect(output).not.toContain("error(s) found"); + }); + it("includes section headers", () => { const result: ValidationResult = { configValid: true, diff --git a/src/validate.ts b/src/validate.ts index a842dfb..0945f24 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -133,6 +133,80 @@ export async function validateConfig( } } + // Step 5: Optional dependency checks + const optionalDepChecks: Array<{ + name: string; + pkg: string; + condition: boolean; + message: string; + }> = []; + + // Check if any source uses type: document + const hasDocumentSource = serverCfg.sources.some( + (s) => s.type === "document", + ); + if (hasDocumentSource) { + // Check for pdf-parse + const hasPdfSources = serverCfg.sources.some( + (s) => + s.type === "document" && + "file_patterns" in s && + (s.file_patterns as string[]).some((p) => p.includes(".pdf")), + ); + if (hasPdfSources) { + try { + await import("pdf-parse"); + } catch { + optionalDepChecks.push({ + name: "pdf-parse", + pkg: "pdf-parse", + condition: true, + message: "Required for PDF document sources", + }); + } + } + // Check for mammoth + const hasDocxSources = serverCfg.sources.some( + (s) => + s.type === "document" && + "file_patterns" in s && + (s.file_patterns as string[]).some((p) => p.includes(".docx")), + ); + if (hasDocxSources) { + try { + await import("mammoth"); + } catch { + optionalDepChecks.push({ + name: "mammoth", + pkg: "mammoth", + condition: true, + message: "Required for DOCX document sources", + }); + } + } + } + + // Check for @xenova/transformers when using local embeddings + if (serverCfg.embedding?.provider === "local") { + try { + await import("@xenova/transformers"); + } catch { + optionalDepChecks.push({ + name: "@xenova/transformers", + pkg: "@xenova/transformers", + condition: true, + message: "Required for local embedding provider", + }); + } + } + + // Add to result + for (const check of optionalDepChecks) { + result.errors.push( + `Missing optional dependency: ${check.name} — ${check.message}. Install: npm install ${check.pkg}`, + ); + } + // Step 4: Tool cross-validation const sourceNames = new Set(serverCfg.sources.map((s) => s.name)); @@ -303,12 +377,32 @@ export function formatValidationResult(result: ValidationResult): string { } lines.push(""); + // Separate optional dep warnings from hard errors for display + const optDepErrors = result.errors.filter((e) => + e.startsWith("Missing optional dependency:"), + ); + const otherErrors = result.errors.filter( + (e) => !e.startsWith("Missing optional dependency:"), + ); + + if (optDepErrors.length > 0) { + lines.push("Optional Dependencies:"); + for (const err of optDepErrors) { + lines.push(` - ${err}`); + } + lines.push(""); + } + const errorCount = result.errors.length; if (errorCount === 0) { lines.push(`Result: All validations passed.`); + } else if (otherErrors.length === 0) { + lines.push( + `Result: ${optDepErrors.length} optional dependency warning(s), no hard errors.`, + ); } else { lines.push(`Result: ${errorCount} error(s) found.`); - for (const err of result.errors) { + for (const err of otherErrors) { lines.push(` - ${err}`); } }