Refactor: Extract DataProvider abstraction from SourceIndexer (#13)

## Summary

- Refactors the monolithic `SourceIndexer` class (~450 lines) into
clean, separated concerns:
- **DataProvider interface** + registry (`src/indexing/providers/`) —
defines the contract between data acquisition and indexing
- **FileDataProvider** — handles git clone/pull, file walking, pattern
matching for all 4 file-based source types
  - **IndexingPipeline** — source-agnostic chunk → embed → upsert logic
- **Shared utilities** (`src/indexing/utils.ts`) — `matchesPatterns`,
`globToRegex`, `hasLowSemanticValue`
- Migrates orchestrator and seed-index script to use the new
abstractions
- Deletes `SourceIndexer` entirely (net -562 lines deleted, +854 added
across 14 files)
- Pure refactor — zero behavioral changes, zero new features

This is Phase 1 of the v1.5 Data Provider Abstraction, preparing the
codebase for API-based sources (Slack, Notion, etc.) in Phase 2.

## Test plan

- [ ] All 534 tests pass locally (54 test files)
- [ ] Build clean (`tsc`, zero errors)
- [ ] Gate 1: New code coexists with SourceIndexer (Tasks 1-4)
- [ ] Gate 2: Orchestrator migrated, full test suite + build passes
(Task 5)
- [ ] Gate 3: Post-merge smoke test against both production instances:
  - `./scripts/smoke-test.sh https://mcp.copilotkit.ai`
  - `./scripts/smoke-test.sh https://mcp.pathfinder.copilotkit.dev`
- Both must show identical behavior (same chunks, search results, tools)

Author: jpr5

scripts/seed-index.ts

@@ -8,7 +8,8 @@
 import { initializeSchema, getPool } from '../src/db/client.js';
 import { getConfig, getServerConfig } from '../src/config.js';
 import { EmbeddingClient } from '../src/indexing/embeddings.js';
-import { SourceIndexer } from '../src/indexing/source-indexer.js';
+import { getProvider } from '../src/indexing/providers/index.js';
+import { IndexingPipeline } from '../src/indexing/pipeline.js';
 
 // ---------------------------------------------------------------------------
 // Arg parsing
@@ -76,17 +77,18 @@ async function main(): Promise<void> {
         const start = Date.now();
         console.log(`--- Indexing source: ${sourceConfig.name} (${sourceConfig.type}) ---`);
 
-        const indexer = new SourceIndexer(
-            sourceConfig,
-            embeddingClient,
-            config.cloneDir,
-            config.githubToken || undefined,
-        );
-
-        await indexer.fullIndex();
+        const provider = getProvider(sourceConfig.type)(sourceConfig, {
+            cloneDir: config.cloneDir,
+            githubToken: config.githubToken || undefined,
+        });
+        const pipeline = new IndexingPipeline(embeddingClient, sourceConfig);
+        const result = await provider.fullAcquire();
+        if (result.items.length > 0) {
+            await pipeline.indexItems(result.items, result.stateToken);
+        }
 
         const elapsed = ((Date.now() - start) / 1000).toFixed(1);
-        console.log(`Source "${sourceConfig.name}" complete in ${elapsed}s\n`);
+        console.log(`Source "${sourceConfig.name}" indexed ${result.items.length} items in ${elapsed}s\n`);
     }
 
     const totalElapsed = ((Date.now() - overallStart) / 1000).toFixed(1);

scripts/test-path-filter.ts

@@ -1,8 +1,8 @@
-// Tests for path filter — verifies include/exclude glob logic in source-indexer
+// Tests for path filter — verifies include/exclude glob logic in indexing utils
 //
 // Usage: npx tsx scripts/test-path-filter.ts
 
-import { globToRegex, matchesPatterns, hasLowSemanticValue } from '../src/indexing/source-indexer.js';
+import { globToRegex, matchesPatterns, hasLowSemanticValue } from '../src/indexing/utils.js';
 import type { SourceConfig } from '../src/types.js';
 
 let passed = 0;

src/__tests__/file-provider.test.ts

@@ -0,0 +1,85 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { FileDataProvider } from '../indexing/providers/file.js';
+import type { SourceConfig } from '../types.js';
+
+describe('FileDataProvider', () => {
+    let tmpDir: string;
+
+    beforeEach(async () => {
+        tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'fp-test-'));
+        await fs.promises.writeFile(path.join(tmpDir, 'readme.md'), '# Hello\nWorld');
+        await fs.promises.writeFile(path.join(tmpDir, 'guide.md'), '# Guide\nContent here');
+        await fs.promises.writeFile(path.join(tmpDir, 'style.css'), 'body { color: red; }');
+        await fs.promises.mkdir(path.join(tmpDir, 'sub'), { recursive: true });
+        await fs.promises.writeFile(path.join(tmpDir, 'sub', 'nested.md'), '# Nested');
+    });
+
+    afterEach(async () => {
+        await fs.promises.rm(tmpDir, { recursive: true, force: true });
+    });
+
+    function makeConfig(overrides?: Partial<SourceConfig>): SourceConfig {
+        return {
+            name: 'test',
+            type: 'markdown',
+            path: tmpDir,
+            file_patterns: ['**/*.md'],
+            chunk: { target_tokens: 600, overlap_tokens: 50 },
+            ...overrides,
+        };
+    }
+
+    it('fullAcquire returns matching files as ContentItems', async () => {
+        const provider = new FileDataProvider(makeConfig(), { cloneDir: '/tmp/test-clones' });
+        const result = await provider.fullAcquire();
+        expect(result.items.length).toBe(3);
+        expect(result.removedIds).toEqual([]);
+        expect(result.stateToken).toMatch(/^local-/);
+        const ids = result.items.map(i => i.id).sort();
+        expect(ids).toContain('readme.md');
+        expect(ids).toContain('guide.md');
+        expect(ids).toContain('sub/nested.md');
+        const readme = result.items.find(i => i.id === 'readme.md');
+        expect(readme?.content).toBe('# Hello\nWorld');
+    });
+
+    it('fullAcquire excludes non-matching patterns', async () => {
+        const provider = new FileDataProvider(makeConfig(), { cloneDir: '/tmp/test-clones' });
+        const result = await provider.fullAcquire();
+        const ids = result.items.map(i => i.id);
+        expect(ids).not.toContain('style.css');
+    });
+
+    it('fullAcquire filters out low-semantic-value content', async () => {
+        const svgContent = 'M0,0 L100,100 C50,50 200.5,300.7 '.repeat(100);
+        await fs.promises.writeFile(path.join(tmpDir, 'data.md'), svgContent);
+        const provider = new FileDataProvider(makeConfig(), { cloneDir: '/tmp/test-clones' });
+        const result = await provider.fullAcquire();
+        const ids = result.items.map(i => i.id);
+        expect(ids).not.toContain('data.md');
+    });
+
+    it('getCurrentStateToken returns local hash for local sources', async () => {
+        const provider = new FileDataProvider(makeConfig(), { cloneDir: '/tmp/test-clones' });
+        const token = await provider.getCurrentStateToken();
+        expect(token).toMatch(/^local-/);
+    });
+
+    it('getCurrentStateToken returns null when path does not exist', async () => {
+        const provider = new FileDataProvider(
+            makeConfig({ path: '/nonexistent/path' }),
+            { cloneDir: '/tmp/test-clones' },
+        );
+        const token = await provider.getCurrentStateToken();
+        expect(token).toBeNull();
+    });
+
+    it('incrementalAcquire falls back to fullAcquire for local sources', async () => {
+        const provider = new FileDataProvider(makeConfig(), { cloneDir: '/tmp/test-clones' });
+        const result = await provider.incrementalAcquire('old-token');
+        expect(result.items.length).toBe(3);
+    });
+});

src/__tests__/pipeline.test.ts

@@ -0,0 +1,111 @@
+import { describe, it, expect, vi } from 'vitest';
+import { IndexingPipeline } from '../indexing/pipeline.js';
+import type { ContentItem } from '../indexing/providers/types.js';
+import type { SourceConfig } from '../types.js';
+
+// Mock the dependencies
+vi.mock('../indexing/chunking/index.js', () => ({
+    getChunker: vi.fn().mockReturnValue(
+        (content: string, _filePath: string, _config: unknown) => [{
+            content,
+            title: 'Test Title',
+            chunkIndex: 0,
+        }],
+    ),
+}));
+
+vi.mock('../indexing/embeddings.js', () => {
+    const MockEmbeddingClient = vi.fn().mockImplementation(function (this: Record<string, unknown>) {
+        this.embedBatch = vi.fn().mockResolvedValue([[0.1, 0.2, 0.3]]);
+    });
+    return { EmbeddingClient: MockEmbeddingClient };
+});
+
+vi.mock('../db/queries.js', () => ({
+    upsertChunks: vi.fn().mockResolvedValue(undefined),
+    deleteChunksByFile: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock('../indexing/url-derivation.js', () => ({
+    deriveUrl: () => 'https://example.com/test',
+}));
+
+const { upsertChunks, deleteChunksByFile } = await import('../db/queries.js');
+const { EmbeddingClient } = await import('../indexing/embeddings.js');
+
+const testConfig: SourceConfig = {
+    name: 'test-source',
+    type: 'markdown',
+    path: 'docs/',
+    file_patterns: ['**/*.md'],
+    chunk: { target_tokens: 600, overlap_tokens: 50 },
+};
+
+describe('IndexingPipeline', () => {
+    it('indexes items: chunk → embed → delete old → upsert', async () => {
+        const embeddingClient = new EmbeddingClient('key', 'model', 1536);
+        const pipeline = new IndexingPipeline(embeddingClient, testConfig);
+
+        const items: ContentItem[] = [{
+            id: 'docs/test.md',
+            content: '# Hello\nSome content here',
+        }];
+
+        await pipeline.indexItems(items, 'abc123');
+
+        expect(deleteChunksByFile).toHaveBeenCalledWith('test-source', 'docs/test.md');
+        expect(upsertChunks).toHaveBeenCalledWith(
+            expect.arrayContaining([
+                expect.objectContaining({
+                    source_name: 'test-source',
+                    file_path: 'docs/test.md',
+                    commit_sha: 'abc123',
+                }),
+            ]),
+        );
+    });
+
+    it('skips items that produce zero chunks', async () => {
+        const { getChunker } = await import('../indexing/chunking/index.js');
+        vi.mocked(getChunker).mockReturnValueOnce(() => []);
+
+        const embeddingClient = new EmbeddingClient('key', 'model', 1536);
+        const pipeline = new IndexingPipeline(embeddingClient, testConfig);
+
+        vi.mocked(upsertChunks).mockClear();
+        await pipeline.indexItems([{ id: 'empty.md', content: '' }], 'abc');
+        expect(upsertChunks).not.toHaveBeenCalled();
+    });
+
+    it('removes items by ID', async () => {
+        const embeddingClient = new EmbeddingClient('key', 'model', 1536);
+        const pipeline = new IndexingPipeline(embeddingClient, testConfig);
+
+        vi.mocked(deleteChunksByFile).mockClear();
+        await pipeline.removeItems(['docs/old.md', 'docs/deleted.md']);
+
+        expect(deleteChunksByFile).toHaveBeenCalledTimes(2);
+        expect(deleteChunksByFile).toHaveBeenCalledWith('test-source', 'docs/old.md');
+        expect(deleteChunksByFile).toHaveBeenCalledWith('test-source', 'docs/deleted.md');
+    });
+
+    it('passes sourceUrl from ContentItem when provided', async () => {
+        const embeddingClient = new EmbeddingClient('key', 'model', 1536);
+        const pipeline = new IndexingPipeline(embeddingClient, testConfig);
+
+        vi.mocked(upsertChunks).mockClear();
+        await pipeline.indexItems([{
+            id: 'docs/test.md',
+            content: 'Content',
+            sourceUrl: 'https://custom.url/test',
+        }], 'abc');
+
+        expect(upsertChunks).toHaveBeenCalledWith(
+            expect.arrayContaining([
+                expect.objectContaining({
+                    source_url: 'https://custom.url/test',
+                }),
+            ]),
+        );
+    });
+});

src/__tests__/provider-registry.test.ts

@@ -0,0 +1,22 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { registerProvider, getProvider } from '../indexing/providers/index.js';
+import type { DataProvider, ProviderOptions } from '../indexing/providers/types.js';
+import type { SourceConfig } from '../types.js';
+
+describe('provider registry', () => {
+    it('registers and retrieves a provider factory', () => {
+        const mockFactory = (config: SourceConfig, options: ProviderOptions): DataProvider => ({
+            fullAcquire: async () => ({ items: [], removedIds: [], stateToken: 'test' }),
+            incrementalAcquire: async () => ({ items: [], removedIds: [], stateToken: 'test' }),
+            getCurrentStateToken: async () => 'test',
+        });
+
+        registerProvider('test-type', mockFactory);
+        const factory = getProvider('test-type');
+        expect(factory).toBe(mockFactory);
+    });
+
+    it('throws for unknown provider type', () => {
+        expect(() => getProvider('nonexistent')).toThrow('Unknown provider type: "nonexistent"');
+    });
+});

src/__tests__/utils.test.ts

@@ -0,0 +1,57 @@
+import { describe, it, expect } from 'vitest';
+import { globToRegex, matchesPatterns, hasLowSemanticValue } from '../indexing/utils.js';
+import type { SourceConfig } from '../types.js';
+
+describe('globToRegex', () => {
+    it('matches ** glob patterns', () => {
+        expect(globToRegex('**/*.ts').test('src/foo.ts')).toBe(true);
+        expect(globToRegex('**/*.ts').test('foo.ts')).toBe(true);
+        expect(globToRegex('**/*.ts').test('foo.js')).toBe(false);
+    });
+
+    it('matches * glob patterns', () => {
+        expect(globToRegex('*.ts').test('foo.ts')).toBe(true);
+        expect(globToRegex('*.ts').test('src/foo.ts')).toBe(false);
+    });
+});
+
+describe('matchesPatterns', () => {
+    const config: SourceConfig = {
+        name: 'test',
+        type: 'code',
+        path: '.',
+        file_patterns: ['**/*.ts', '**/*.tsx'],
+        exclude_patterns: ['**/test/**', '**/*.test.*'],
+        chunk: { target_lines: 80, overlap_lines: 10 },
+    };
+
+    it('includes matching files', () => {
+        expect(matchesPatterns('src/index.ts', config)).toBe(true);
+        expect(matchesPatterns('src/deep/path/file.tsx', config)).toBe(true);
+    });
+
+    it('excludes matching patterns', () => {
+        expect(matchesPatterns('src/test/helper.ts', config)).toBe(false);
+        expect(matchesPatterns('src/foo.test.ts', config)).toBe(false);
+    });
+
+    it('rejects non-matching extensions', () => {
+        expect(matchesPatterns('src/index.js', config)).toBe(false);
+    });
+});
+
+describe('hasLowSemanticValue', () => {
+    it('returns false for short content', () => {
+        expect(hasLowSemanticValue('short')).toBe(false);
+    });
+
+    it('returns true for SVG-like content', () => {
+        const svgData = 'M0,0 L100,100 C50,50 200.5,300.7 '.repeat(100);
+        expect(hasLowSemanticValue(svgData)).toBe(true);
+    });
+
+    it('returns false for normal text', () => {
+        const text = 'This is a normal document with some text content that describes how things work. '.repeat(20);
+        expect(hasLowSemanticValue(text)).toBe(false);
+    });
+});