From 6b48b4b416299e3362eb43552479f3524059cb7c Mon Sep 17 00:00:00 2001 From: Brian Jones Date: Fri, 15 May 2026 13:01:34 -0500 Subject: [PATCH] fix(SemanticMemory): pre-load sqlite-vec before probe; skip virtual tables with missing modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The secondary probe-read in `SemanticMemory.open()` would treat "no such module: vec0" as corruption, quarantining the DB and producing an infinite ~60s recovery loop whenever the on-disk schema contained the `entity_embeddings` vec0 virtual table (observed ~1,400 quarantine files/day in a long-running deployment, ~610 MB peak). The bug is load-order, not missing-package: sqlite-vec is installed, but the probe runs synchronously in `open()` before any deferred extension load can happen. The next open finds the DB it just created, the vec0 table is still there, the probe fails again, and the cycle repeats. Two changes: 1. Pre-load `sqlite-vec` directly (not via `EmbeddingProvider`) immediately after `constructor(this.config.dbPath)`. The provider may not be attached at open() time, but the on-disk schema doesn't know about that runtime choice — the probe needs vec0 queryable regardless. 2. Per-table missing-module guard in the probe: if a row in `sqlite_master` is a `CREATE VIRTUAL TABLE` and the probe throws "no such module", skip it. Treating a missing extension as corruption is the wrong dispatch — the table isn't broken, its module just isn't here. Both changes are independently sufficient for the vec0 case. Together they cover sqlite-vec being unavailable at all (the guard handles it) and future virtual-table modules that follow the same pattern. Tests cover the load-order contract: - Healthy DB with vec0 virtual table opens without quarantine - Repeated opens accumulate zero quarantine artifacts - DB opens cleanly even with no embedding provider attached - Genuine probe-detected corruption (torn interior page) still quarantines — regression guard for the carve-out Red-green verified: with the source change reverted, the three new probe tests fail; with it applied, all 16 tests in the file pass (13 existing + 3 new). The full SemanticMemory unit suite (114 tests across 4 files) is green. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/memory/SemanticMemory.ts | 37 ++++++- ...emantic-memory-corruption-recovery.test.ts | 104 ++++++++++++++++++ 2 files changed, 139 insertions(+), 2 deletions(-) diff --git a/src/memory/SemanticMemory.ts b/src/memory/SemanticMemory.ts index a1f19d23a..1ca0c4468 100644 --- a/src/memory/SemanticMemory.ts +++ b/src/memory/SemanticMemory.ts @@ -567,6 +567,25 @@ export class SemanticMemory { this.db = constructor(this.config.dbPath) as Database; + // Pre-load sqlite-vec so vec0 virtual tables are queryable during the probe + // below. Without this, an existing entity_embeddings (vec0) table — which is + // recreated asynchronously by initializeVectorSearch() the first time vector + // search runs — causes the probe to throw "no such module: vec0" and triggers + // false-positive corruption quarantine. The probe runs synchronously in open() + // before the async vector-search init has a chance to load the extension. + // + // Loading here uses sqlite-vec directly (not via EmbeddingProvider) because: + // 1. EmbeddingProvider may not be attached at open() time. + // 2. The probe must succeed regardless of whether vector search is wired up, + // since the on-disk DB schema doesn't know about that runtime choice. + // + // Failure is non-fatal: the probe loop below has its own missing-module guard + // that skips virtual tables whose extension isn't available. + try { + const vec = await import('sqlite-vec'); + vec.load(this.db); + } catch { /* @silent-fallback-ok: sqlite-vec unavailable — probe will skip vec0 tables */ } + // Integrity check — auto-recover from corruption (JSONL is source of truth). // Corrupt DBs are quarantined (renamed) not deleted, and a marker file is written // so operators can notice the recovery after the fact. @@ -589,9 +608,23 @@ export class SemanticMemory { if (!this._needsRebuild) { try { const tables = this.db!.prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'fts%' AND name NOT LIKE 'sqlite%'" - ).all() as Array<{ name: string }>; + "SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'fts%' AND name NOT LIKE 'sqlite%'" + ).all() as Array<{ name: string; sql: string | null }>; for (const t of tables) { + // Virtual tables require their module to be loaded into the connection. + // If a virtual-table probe throws "no such module", the table isn't + // corrupt — its extension just isn't available (sqlite-vec missing, + // a custom module not yet registered, etc.). Treating that as + // corruption produces an infinite quarantine loop on every open. + if (t.sql && /^\s*CREATE\s+VIRTUAL\s+TABLE/i.test(t.sql)) { + try { + this.db!.prepare(`SELECT * FROM "${t.name}" LIMIT 100`).all(); + } catch (vErr) { + if (/no such module/i.test((vErr as Error).message)) continue; + throw vErr; + } + continue; + } this.db!.prepare(`SELECT * FROM "${t.name}" LIMIT 100`).all(); } } catch (err) { diff --git a/tests/unit/semantic-memory-corruption-recovery.test.ts b/tests/unit/semantic-memory-corruption-recovery.test.ts index 438e03bdf..e8267a5c5 100644 --- a/tests/unit/semantic-memory-corruption-recovery.test.ts +++ b/tests/unit/semantic-memory-corruption-recovery.test.ts @@ -22,6 +22,7 @@ import fs from 'node:fs'; import path from 'node:path'; import os from 'node:os'; import { SemanticMemory } from '../../src/memory/SemanticMemory.js'; +import { EmbeddingProvider } from '../../src/memory/EmbeddingProvider.js'; interface Setup { dir: string; @@ -310,3 +311,106 @@ describe('SemanticMemory corruption auto-recovery', () => { expect(afterSecond.corruptFiles.length).toBe(beforeSecond.corruptFiles.length); }); }); + +/** + * Virtual-table probe load-order tests. + * + * Contract: + * The secondary probe-read MUST NOT treat "no such module: " on a + * virtual table as corruption. Virtual tables require their extension to be + * loaded into the connection before they're queryable, and the probe runs + * synchronously in open() before any deferred extension load could happen. + * Treating a missing-module error as corruption produces an infinite + * quarantine loop on every reopen (see the upstream gap report). + * + * Concretely for sqlite-vec: open() pre-loads the extension when available, + * so a healthy DB with an `entity_embeddings` (vec0) virtual table opens + * without quarantine. + */ +describe('SemanticMemory probe — virtual tables and load order', () => { + let setup: Setup; + beforeEach(() => { setup = makeSetup(); }); + afterEach(() => setup.cleanup()); + + async function seedDbWithVec0VirtualTable(dbPath: string): Promise { + // Seed the DB exactly the way SemanticMemory's hybrid-search path does: open + // through SemanticMemory with an EmbeddingProvider attached, initialize vector + // search (which loads sqlite-vec and creates the `entity_embeddings` vec0 + // virtual table via VectorSearch.createTable), then close. The on-disk schema + // now has a vec0 virtual table that requires the sqlite-vec module to query. + const memory = new SemanticMemory({ dbPath, decayHalfLifeDays: 30, lessonDecayHalfLifeDays: 90, staleThreshold: 0.2 }); + memory.setEmbeddingProvider(new EmbeddingProvider()); + await memory.open(); + await memory.initializeVectorSearch(); + memory.close(); + } + + it('does not quarantine a healthy DB that contains a vec0 virtual table', async () => { + await seedDbWithVec0VirtualTable(setup.dbPath); + + // Sanity: schema actually contains a CREATE VIRTUAL TABLE for entity_embeddings. + const BetterSqlite3 = (await import('better-sqlite3')).default; + const inspect = new BetterSqlite3(setup.dbPath, { readonly: true }); + const row = inspect.prepare( + "SELECT name, sql FROM sqlite_master WHERE name='entity_embeddings'" + ).get() as { name: string; sql: string } | undefined; + inspect.close(); + expect(row).toBeDefined(); + expect(row!.sql).toMatch(/CREATE\s+VIRTUAL\s+TABLE/i); + + // Reopen via SemanticMemory — must not throw and must not quarantine. + const mem = new SemanticMemory({ dbPath: setup.dbPath, decayHalfLifeDays: 30, lessonDecayHalfLifeDays: 90, staleThreshold: 0.2 }); + await expect(mem.open()).resolves.not.toThrow(); + mem.close(); + + const { corruptFiles, markerFiles } = listCorruptArtifacts(setup.dir); + expect(corruptFiles).toEqual([]); + expect(markerFiles).toEqual([]); + }); + + it('opening repeatedly does not accumulate quarantine artifacts (no probe-loop regression)', async () => { + await seedDbWithVec0VirtualTable(setup.dbPath); + + for (let i = 0; i < 3; i++) { + const mem = new SemanticMemory({ dbPath: setup.dbPath, decayHalfLifeDays: 30, lessonDecayHalfLifeDays: 90, staleThreshold: 0.2 }); + await mem.open(); + mem.close(); + } + + const { corruptFiles, markerFiles } = listCorruptArtifacts(setup.dir); + expect(corruptFiles).toEqual([]); + expect(markerFiles).toEqual([]); + }); + + it('opens cleanly with no embedding provider attached (probe still survives the vec0 schema)', async () => { + // The deferred-attachment shape: vector search may not be wired up on every + // reopen, but the on-disk schema still contains the vec0 virtual table from + // a previous run. The pre-load + per-table missing-module guard must cover + // this path so we don't quarantine. + await seedDbWithVec0VirtualTable(setup.dbPath); + + const mem = new SemanticMemory({ dbPath: setup.dbPath, decayHalfLifeDays: 30, lessonDecayHalfLifeDays: 90, staleThreshold: 0.2 }); + // Intentionally no setEmbeddingProvider() — exercise the "vec0 table exists + // on disk but caller hasn't asked for vector search" reopen path. + await mem.open(); + mem.close(); + + const { corruptFiles, markerFiles } = listCorruptArtifacts(setup.dir); + expect(corruptFiles).toEqual([]); + expect(markerFiles).toEqual([]); + }); + + it('genuine probe-detected corruption still quarantines (regression guard for the virtual-table carve-out)', async () => { + // Make sure the virtual-table skip didn't accidentally widen the probe's + // tolerance for actual corruption in non-virtual tables. + await writePartiallyCorruptDb(setup.dbPath); + + const mem = new SemanticMemory({ dbPath: setup.dbPath, decayHalfLifeDays: 30, lessonDecayHalfLifeDays: 90, staleThreshold: 0.2 }); + await mem.open(); + mem.close(); + + const { corruptFiles, markerFiles } = listCorruptArtifacts(setup.dir); + expect(corruptFiles.length).toBeGreaterThanOrEqual(1); + expect(markerFiles.length).toBe(1); + }); +});