Repository navigation
Expand file tree
/
Copy pathdb.ts
More file actions
872 lines (796 loc) · 70.7 KB
/
Copy pathdb.ts
File metadata and controls
872 lines (796 loc) · 70.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
/** SQLite connection management and index-DB schema. Ports the connection-pragma setup from `db.py` (WAL journal mode, NORMAL synchronous) and the index schema (files / symbols / refs / FTS5) that later layers query. Each database file gets one lazily-opened, cached connection. The connection itself comes from `./sqlite_driver.js`, a thin better-sqlite3-shaped facade over Node's built-in `node:sqlite`; nothing in this file talks to `node:sqlite` directly. */
import * as fs from 'node:fs'
import { createRequire } from 'node:module'
import * as path from 'node:path'
import Database from './sqlite_driver.js'
import type { SqliteDatabase } from './sqlite_driver.js'
import { dataDir, globalDbPath, SYMBOL_BODY_CHAR_CAP } from './constants.js'
import { isDotenvPath } from './dotenv_redact.js'
import { safeJoin } from './paths.js'
import { projectScopeClause } from './sql_path.js'
import { ensureDirSync, foldCase, foldPath, normalizePath, sleepSync } from './util.js'
import { registerReset } from './reset.js'
// ESM has no `require`; build one so we can probe for the optional sqlite-vec package without making it a hard module-resolution dependency.
const _require = createRequire(import.meta.url)
// One Database handle per absolute db path. Keyed by the resolved path so two callers naming the same file via different relative strings share a handle.
const _connections = new Map<string, SqliteDatabase>()
/** Index-DB schema (matches the spec for Layer 2). Tables: - files — one row per indexed source file. - symbols — extracted definitions (functions, classes, types, ...). - refs — references/usages of names, for caller lookups. - chunks — semantic search chunk metadata (filePath, startLine, endLine, text, kind). - notes — file/symbol-attached architecture notes with a staleness fingerprint (notes.ts). - outline_listing_cache — bounded, redacted outline-listing cache. - symbols_fts — FTS5 mirror of symbols for full-text name/body search. The FTS5 table is content-linked to `symbols` (external-content) so the row data lives once in `symbols`; triggers keep the index in sync on write. `index_retries` holds the worker's transient-read-failure counters instead of a column on `files`. The count has to survive across processes -- the edit hook is a short-lived CLI process, the drain loop a long-lived daemon -- so it has to be on disk in the DB they share. Putting it in `files` meant minting a row there for a path that had never been indexed, and a `files` row is what three readers take to mean "this file is in the index": indexMatchesDisk (index_freshness.ts) and healStaleIndex/staleWarning (read_commands.ts) each read a row with no sha as a pre-fingerprinting legacy row and accept it as-is, so a never-indexed file that hit one transient lock during a drain looked indexed and lost its on-demand self-heal for good. `files.retry_count` is still created for older databases and is no longer read or written. Prose belongs in this comment rather than in an SQL `--` line inside the template literal below: a comment here is stripped by the bundler, while one inside the string ships verbatim in every hook process's eager chunk set. tests/guards/dist_chunks_deduped.test.ts holds the ceiling that catches the difference. */
const SCHEMA_SQL = `
CREATE TABLE IF NOT EXISTS files (
path TEXT PRIMARY KEY,
sha TEXT,
mtime REAL,
language TEXT,
indexed_at REAL,
embed_sha TEXT,
retry_count INTEGER NOT NULL DEFAULT 0,
parser_sha TEXT
);
-- Expression index on TG_LOWER(path) -- see pathEqClause (sql_path.ts) and TG_LOWER's
-- registration above. TG_LOWER is registered { deterministic: true }, which is required for
-- SQLite to index an expression at all; without it CREATE INDEX on a function call throws
-- "non-deterministic functions prohibited in index expressions". Because pathEqClause emits
-- this exact 'TG_LOWER(path) = ?' text for every case-insensitive-filesystem query, the planner
-- matches it against this index and uses SEARCH instead of a full table SCAN, without requiring
-- any writer to populate a separate folded column (verified via EXPLAIN QUERY PLAN in
-- db.test.ts / sql_path.test.ts). CREATE INDEX IF NOT EXISTS is purely additive and safe to run
-- against an already-populated table on every connection open, unlike an ALTER TABLE column add
-- -- no MIGRATIONS entry or SCHEMA_VERSION bump is needed for this index.
CREATE INDEX IF NOT EXISTS idx_files_path_folded ON files(TG_LOWER(path));
-- Transient-read-failure counters, keyed by path. See the note above SCHEMA_SQL for why they are not in files.
CREATE TABLE IF NOT EXISTS index_retries (
path TEXT PRIMARY KEY,
retry_count INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_index_retries_path_folded ON index_retries(TG_LOWER(path));
CREATE TABLE IF NOT EXISTS symbols (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_path TEXT,
name TEXT,
kind TEXT,
line_start INTEGER,
line_end INTEGER,
body TEXT,
docstring TEXT,
parent TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name);
CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(file_path);
CREATE INDEX IF NOT EXISTS idx_symbols_name_kind ON symbols(name, kind);
CREATE INDEX IF NOT EXISTS idx_symbols_file_folded ON symbols(TG_LOWER(file_path));
CREATE INDEX IF NOT EXISTS idx_symbols_file_name_folded ON symbols(TG_LOWER(file_path), name);
CREATE INDEX IF NOT EXISTS idx_symbols_call_graph_scope ON symbols(TG_LOWER(file_path), name, line_start DESC, id ASC);
CREATE INDEX IF NOT EXISTS idx_symbols_call_graph_enclosing ON symbols(TG_LOWER(file_path), line_start DESC, id ASC);
-- Partial index backing checkSymbolBodySize (symbol_body_probe.ts), which every SessionStart hook runs.
-- Its predicate cannot be served by any index above, so the check had to read the whole symbols
-- table -- 226 MB / 231324 rows here, 229 ms per session start, and the early-exit LIMIT 1 never
-- fires on a healthy index because there is nothing to find. Indexing the *violating* rows only
-- makes the check a lookup into a b-tree that is empty on a healthy index: measured 229 ms -> 0.0
-- ms, 4 KB on disk, and no measurable insert cost (-0.2%, within noise, over 40000 real rows),
-- because SQLite evaluates the predicate and skips the b-tree write for every row under the cap.
-- SQLite uses a partial index only where the query's WHERE implies the index's, so the probe in
-- symbol_body_probe.ts spells its comparison the same way and against the same constant. That makes the
-- threshold part of the stored schema -- see SYMBOL_BODY_CHAR_CAP in constants.ts for what
-- changing it requires. A query with a lower threshold correctly gets a full scan instead, so no
-- other reader can be served stale rows by this index.
CREATE INDEX IF NOT EXISTS idx_symbols_oversized_body ON symbols(id) WHERE LENGTH(body) > ${SYMBOL_BODY_CHAR_CAP};
CREATE TABLE IF NOT EXISTS refs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_path TEXT,
name TEXT,
line INTEGER,
col INTEGER,
context TEXT
);
CREATE INDEX IF NOT EXISTS idx_refs_name ON refs(name);
CREATE INDEX IF NOT EXISTS idx_refs_file ON refs(file_path);
CREATE INDEX IF NOT EXISTS idx_refs_file_folded ON refs(TG_LOWER(file_path));
CREATE INDEX IF NOT EXISTS idx_refs_file_name_folded ON refs(TG_LOWER(file_path), name);
-- name is the callee and context the caller for tree-sitter refs. These covering indexes are the two directions of the graph matrix; file/line/column ordering stays inside SQLite instead of triggering per-file symbol lookups.
CREATE INDEX IF NOT EXISTS idx_refs_call_graph_callee ON refs(name, file_path, line, col, context);
CREATE INDEX IF NOT EXISTS idx_refs_call_graph_caller ON refs(context, TG_LOWER(file_path), line, col, name);
CREATE TABLE IF NOT EXISTS chunks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_path TEXT,
start_line INTEGER,
end_line INTEGER,
text TEXT,
kind TEXT
);
CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path);
CREATE INDEX IF NOT EXISTS idx_chunks_file_folded ON chunks(TG_LOWER(file_path));
CREATE INDEX IF NOT EXISTS idx_chunks_file_kind_folded ON chunks(TG_LOWER(file_path), kind);
-- Tracks every project root a hook has ever seen an edit for, so the worker's periodic sweep
-- (sweepKnownRoots in index_prune.ts) knows which roots to auto-prune without scanning the
-- entire shared files table for distinct top-level directories on every cycle. Purely additive
-- (no SCHEMA_VERSION bump needed): last_seen_ms is refreshed on every observed edit,
-- first_missing_ms is set the first sweep that finds the root unreachable and cleared the
-- moment it's seen reachable again -- see sweepKnownRoots' grace-period logic.
CREATE TABLE IF NOT EXISTS known_roots (
root TEXT PRIMARY KEY,
last_seen_ms REAL NOT NULL,
first_missing_ms REAL
);
-- Resume points for a budget-truncated reconcile sweep (reconcile.ts), up to four rows per project root (the swept and slowlist ones, reconcile_stamp.ts, keep a time in updated_at). The row keyed by the root itself is the scan's: a project too large to finish a sweep inside DEFAULT_RECONCILE_BUDGET_MS would otherwise scan the same deterministic (git ls-files) prefix every session forever, leaving every file after the budget cutoff permanently unchecked. Its last_scanned_path is the last tracked file the sweep finished examining before its budget ran out; the next sweep rotates its scan order to resume right after that file, wrapping back to the start. The row keyed by the root, a NUL character and the word deletions is the deletion pass's: its last_scanned_path is the last indexed row, in sorted folded-path order, whose file the pass statted before the budget ran out, and the next pass resumes after it, wrapping. Either row is cleared (row deleted) the moment its pass completes a full lap without running out of budget, and both are purged when the project leaves the index (index_prune.ts purgeReconcileCursors: project exclude and a known root that stayed unreachable past its grace). Purely additive (no SCHEMA_VERSION bump needed): a missing row just means "start from the beginning", the same as a fresh database.
CREATE TABLE IF NOT EXISTS reconcile_cursor (
root TEXT PRIMARY KEY,
last_scanned_path TEXT NOT NULL,
updated_at REAL NOT NULL
);
-- Projects indexed on purpose, see indexed_roots.ts.
CREATE TABLE IF NOT EXISTS indexed_roots (
root TEXT PRIMARY KEY,
indexed_ms REAL NOT NULL
);
-- Cross-cache full-text search index for 'token-goat recall' (recall_index.ts). One row
-- per bash-output/web-output/mcp-output blob-store entry (see disk_cache.ts), refreshed
-- in place (ON CONFLICT DO UPDATE) whenever storeBashOutput/storeWebOutput/storeMcpOutput
-- write that entry, so recall never needs a separate rebuild step. row_id is a plain
-- surrogate integer key -- entry_id is the real blob-store id (bash/mcp ids are hex,
-- web ids are the cache's own scheme) and is not unique on its own since bash-output and
-- mcp-output ids share one namespace (BASH_OUTPUT_SUBDIR) while web-output ids are a
-- separate namespace; cache_type disambiguates.
CREATE TABLE IF NOT EXISTS cache_recall (
row_id INTEGER PRIMARY KEY AUTOINCREMENT,
cache_type TEXT NOT NULL,
entry_id TEXT NOT NULL,
label TEXT,
content TEXT,
stored_at REAL,
UNIQUE(cache_type, entry_id)
);
CREATE INDEX IF NOT EXISTS idx_cache_recall_type ON cache_recall(cache_type);
CREATE INDEX IF NOT EXISTS idx_cache_recall_type_stored ON cache_recall(cache_type, stored_at DESC);
CREATE TABLE IF NOT EXISTS outline_listing_cache (
cache_version INTEGER NOT NULL,
cache_key TEXT NOT NULL,
payload TEXT NOT NULL,
PRIMARY KEY (cache_version, cache_key)
);
-- Per-emission ledger for 'token-goat hint-stats' (hint_stats.ts). One row per hint
-- emission event (a hook returning a 'context' HookOutput classified as a discretionary
-- efficiency nudge, as opposed to a mandatory informational injection -- see hint_stats.ts's
-- doc comment for the exact category list and what is deliberately excluded). correlator is a
-- file-path/output-id pointer the hint carries, used to check
-- whether a later Bash tool call in the same session actually followed the hint's specific
-- pointer (see resolvePendingHintsForEvent). Supplied by the hint builder itself via
-- HookOutput's correlators field wherever one exists, and only fallen back to a regex scrape
-- of the rendered hint text (hint_stats.ts's extractPathCorrelator, whose doc comment records
-- what that scrape actually measured) where it does not. A row can name SEVERAL files -- the
-- sed/awk line-range hint joins one per file into one emission -- stored newline-separated and
-- matched on any member; see joinCorrelators. NULL when no pointer could be supplied or extracted,
-- in which case the row is inserted already resolved and observable=0: there is nothing a later
-- command could do that this row would recognize, so no verdict about it was ever observed.
-- observable=1 is the normal case. Efficacy counts only observable rows, numerator and denominator
-- both, because an unobservable row's acted_on is a fabrication in whichever direction the code
-- picked and averaging fabrications into a measured rate is how 174 of bash_redirect's 698 rows
-- came to hold its efficacy down to 0.86% when the observed figure was 1.15%. calls_remaining is
-- the countdown of subsequent tool-use events still eligible to resolve this row before it is
-- considered timed out. A displayed, observable row with resolved=0 is pending: nothing has
-- scored it yet. Nothing sweeps these on a clock, so a session that ends mid-window leaves its
-- rows pending for good, and efficacy and suppression count only resolved rows -- counting a
-- pending one as not acted on is what muted bash_redirect at 1 in 11 when its scored rows stood
-- at 1 in 6. displayed=0 marks a detection that never reached the agent -- auto-
-- suppressed, or declined by a hint's own net-benefit gate -- recorded so the ledger can answer
-- how often a hint fired at all, which it previously could not: a suppressed detection left no
-- trace, so a fully-muted category was indistinguishable from one that had stopped triggering.
-- Such a row is always observable=0 and bytes_emitted=0: nothing was shown, so nothing was spent
-- and no verdict was available. It shares hint_emissions' 180-day retention (pruneHintEmissions
-- in stats.ts deletes on emitted_at alone), deliberately unlike hint_suppression_probes, which is
-- one upserted row per key rather than a growing log and is therefore never pruned.
CREATE TABLE IF NOT EXISTS hint_emissions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
session_id TEXT NOT NULL,
harness TEXT NOT NULL,
correlator TEXT,
emitted_at REAL NOT NULL,
resolved INTEGER NOT NULL DEFAULT 0,
acted_on INTEGER NOT NULL DEFAULT 0,
calls_remaining INTEGER NOT NULL DEFAULT 0,
bytes_emitted INTEGER,
observable INTEGER NOT NULL DEFAULT 1,
displayed INTEGER NOT NULL DEFAULT 1
);
CREATE INDEX IF NOT EXISTS idx_hint_emissions_category ON hint_emissions(category);
CREATE INDEX IF NOT EXISTS idx_hint_emissions_session_pending ON hint_emissions(session_id, resolved);
CREATE INDEX IF NOT EXISTS idx_hint_emissions_cat_harness ON hint_emissions(category, harness);
-- Manual efficacy votes for a hint category (token-goat hint-stats --mark-effective/--mark-ineffective),
-- kept separate from hint_emissions' automatic acted_on signal so the two are never silently
-- blended -- see hint_stats.ts's doc comment on why some categories only support this manual signal.
CREATE TABLE IF NOT EXISTS hint_manual_marks (
category TEXT PRIMARY KEY,
effective_count INTEGER NOT NULL DEFAULT 0,
ineffective_count INTEGER NOT NULL DEFAULT 0
);
-- Durable counter backing hint_stats.ts's backoff-threshold probe-recovery schedule: how many
-- CONSECUTIVE suppressed occasions have elapsed for (category, harness) since a hint in this
-- category was last actually shown (either organically, because shouldSuppress no longer holds,
-- or via a prior probe). shouldSuppress itself stays a pure function of hint_emissions -- this
-- table exists only because a suppressed occasion is deliberately never written to
-- hint_emissions (see that table's own comment), so without a separate durable counter here
-- there would be no way to know "how many suppressed occasions have we seen" across the
-- short-lived hook CLI processes that call applyHintTracking. Keyed by (category, harness), not
-- category alone, to match shouldSuppress/categoryStats' own per-harness scoping -- unlike
-- hint_manual_marks (a human-entered vote, deliberately not harness-split).
CREATE TABLE IF NOT EXISTS hint_suppression_probes (
category TEXT NOT NULL,
harness TEXT NOT NULL,
streak INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (category, harness)
);
-- Free-text architecture/rationale notes (the "why" layer -- see notes.ts), attached either to
-- a whole file (symbol = '') or to one specific indexed symbol within it (symbol = that
-- symbol's name). '' rather than NULL for the whole-file case because SQLite's UNIQUE treats
-- NULLs as pairwise-distinct (never conflicting with each other), which would let note-add
-- accumulate unlimited duplicate whole-file notes for the same file instead of upserting one;
-- '' is a real, comparable value so UNIQUE(file_path, symbol) enforces "at most one note per
-- attachment point" for both cases identically. 'fingerprint' is a SHA-256 digest (see
-- fingerprintContent in fingerprint.ts) captured at write time of exactly what the note
-- describes -- the resolved symbol's current body text for a symbol-scoped note, or a stable
-- digest of the file's current top-level symbol manifest (name:kind:line-range per symbol,
-- sorted) for a file-scoped note -- so 'token-goat note-list --stale-only' can recompute the
-- same fingerprint against the live index later and flag a mismatch (see notes.ts's
-- isNoteStale). Staleness detection is purely advisory: nothing here ever auto-rewrites or
-- deletes a note's content, only flags that the code it describes has moved since it was
-- written -- a human/agent re-review decides what to do with a stale note.
CREATE TABLE IF NOT EXISTS notes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_path TEXT NOT NULL,
symbol TEXT NOT NULL DEFAULT '',
content TEXT NOT NULL,
fingerprint TEXT NOT NULL,
created_at REAL NOT NULL,
updated_at REAL NOT NULL,
UNIQUE(file_path, symbol)
);
CREATE INDEX IF NOT EXISTS idx_notes_file_folded ON notes(TG_LOWER(file_path));
-- Baseline for skill_version_drift.ts's one-shot nudge: the token-goat CLI version (and its
-- flat command-name set, JSON-encoded) active the moment the token-goat skill's body was
-- last (re)loaded into this session -- see hooks_skill.ts's postSkillHandler. A session that
-- keeps running after the CLI is upgraded has no other way to learn that new surgical-read
-- commands now exist (the skill only re-announces itself on an explicit reload), so
-- checkSkillVersionDrift compares this snapshot against the live command set on each user turn
-- and fires the nudge exactly once (notified_at) per (re)load. session_id is the primary key
-- because only one skill (token-goat) is ever tracked here.
CREATE TABLE IF NOT EXISTS skill_version_snapshots (
session_id TEXT PRIMARY KEY,
skill_name TEXT NOT NULL,
loaded_version TEXT NOT NULL,
loaded_commands_json TEXT NOT NULL,
notified_at REAL
);
-- Which embedding stack produced the vectors currently in chunk_vectors -- the model, its
-- pinned revision, and the inference runtime (see embeddingProvenance in embeddings.ts). The
-- vector table itself is vec0(rowid, embedding) and has nowhere to record this, so without
-- this row a database that was embedded by one stack and then added to by another holds two
-- incomparable sets of vectors under one index, with nothing able to tell them apart. That is
-- not hypothetical: global.db is machine-wide across every project on the machine (see
-- constants.ts), so upgrading the runtime, or changing the model or its pinned revision, mixes
-- old and new vectors for as long as the old files go untouched. Measured drift between two
-- runtime versions of the same quantized model is 0.9925-0.9978 cosine on the final vector --
-- small, but enough to reorder near-ties, and invisible to every existing check.
--
-- Single-row by construction (the CHECK pins the key), because there is exactly one vector
-- table per database. An EMPTY table on a database that already holds chunks means the vectors
-- predate this stamp and their provenance is unknowable -- see ensureEmbeddingProvenance, which
-- treats that exactly like a mismatch. That is what makes this work without a migration step.
CREATE TABLE IF NOT EXISTS embedding_provenance (
id INTEGER PRIMARY KEY CHECK (id = 1),
provenance TEXT NOT NULL
);
`
// FTS5 is a compile-time-optional SQLite extension. Node's bundled SQLite ships with it enabled, but wrap creation so a build without FTS5 still yields a usable (search-degraded) index DB rather than throwing on open.
/** The tokenizer both FTS5 mirrors are declared with. `remove_diacritics 2` rather than FTS5's default of 1: the default folds only diacritics encoded as a single precomposed codepoint, so `café` was already findable as `cafe` before this was set, but `Hà Nội` was not findable as `Ha Noi` (https://sqlite.org/fts5.html#unicode61_tokenizer). Named as a constant because {@link rebuildFtsAtCurrentTokenizer} has to compare a database's stored declaration against it, and a second copy of the string is exactly how those two drift apart. */
export const FTS_TOKENIZER = 'unicode61 remove_diacritics 2'
// The `tokenize=` argument here reaches a database exactly once, when its virtual table is first created: `CREATE VIRTUAL TABLE IF NOT EXISTS` against an existing table of the same name is a silent no-op, and SQLite reports no error and no warning when the stored declaration differs from this one. Changing the tokenizer therefore needs a SCHEMA_VERSION bump and a step in MIGRATIONS that drops, re-creates and rebuilds both tables -- see MIGRATIONS[13] -- or the change reaches only databases created after it, leaving two populations that search differently with nothing to tell them apart. tests/guards/fts_tokenizer_reaches_existing_databases.test.ts fails if this literal moves without that bump.
const FTS_SQL = `
CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(
name,
body,
docstring,
content='symbols',
content_rowid='id',
tokenize='${FTS_TOKENIZER}'
);
CREATE TRIGGER IF NOT EXISTS symbols_ai AFTER INSERT ON symbols BEGIN
INSERT INTO symbols_fts(rowid, name, body, docstring)
VALUES (new.id, new.name, new.body, new.docstring);
END;
CREATE TRIGGER IF NOT EXISTS symbols_ad AFTER DELETE ON symbols BEGIN
INSERT INTO symbols_fts(symbols_fts, rowid, name, body, docstring)
VALUES ('delete', old.id, old.name, old.body, old.docstring);
END;
CREATE TRIGGER IF NOT EXISTS symbols_au AFTER UPDATE ON symbols BEGIN
INSERT INTO symbols_fts(symbols_fts, rowid, name, body, docstring)
VALUES ('delete', old.id, old.name, old.body, old.docstring);
INSERT INTO symbols_fts(rowid, name, body, docstring)
VALUES (new.id, new.name, new.body, new.docstring);
END;
-- Content-linked FTS5 mirror of cache_recall (recall_index.ts), same shape as symbols_fts
-- above. An INSERT ... ON CONFLICT DO UPDATE against cache_recall fires the AFTER UPDATE
-- trigger (not AFTER INSERT) on the conflicting row, same as any other SQLite upsert, so the
-- delete+reinsert pattern below keeps the fts index correct on a re-indexed (overwritten)
-- entry, not just a brand-new one.
CREATE VIRTUAL TABLE IF NOT EXISTS cache_recall_fts USING fts5(
label,
content,
content='cache_recall',
content_rowid='row_id',
tokenize='${FTS_TOKENIZER}'
);
CREATE TRIGGER IF NOT EXISTS cache_recall_ai AFTER INSERT ON cache_recall BEGIN
INSERT INTO cache_recall_fts(rowid, label, content)
VALUES (new.row_id, new.label, new.content);
END;
CREATE TRIGGER IF NOT EXISTS cache_recall_ad AFTER DELETE ON cache_recall BEGIN
INSERT INTO cache_recall_fts(cache_recall_fts, rowid, label, content)
VALUES ('delete', old.row_id, old.label, old.content);
END;
CREATE TRIGGER IF NOT EXISTS cache_recall_au AFTER UPDATE ON cache_recall BEGIN
INSERT INTO cache_recall_fts(cache_recall_fts, rowid, label, content)
VALUES ('delete', old.row_id, old.label, old.content);
INSERT INTO cache_recall_fts(rowid, label, content)
VALUES (new.row_id, new.label, new.content);
END;
`
// Bump this the day SCHEMA_SQL changes in a way `CREATE TABLE IF NOT EXISTS` can't express on an already-populated table -- a column add/rename/drop, a type change, a data backfill -- and add the matching step to MIGRATIONS below. It represents the schema as it exists today. v3 -> v4: added cache_recall / cache_recall_fts (token-goat recall). Purely additive -- `CREATE TABLE/VIRTUAL TABLE IF NOT EXISTS` in SCHEMA_SQL/FTS_SQL already handles a pre-existing v3 database, so no MIGRATIONS[3] step is needed (same reasoning as the comment on MIGRATIONS below for a bump with no registered step). v4 -> v5: added hint_emissions / hint_manual_marks (token-goat hint-stats). Purely additive -- `CREATE TABLE IF NOT EXISTS` in SCHEMA_SQL already handles a pre-existing v4 database, so no MIGRATIONS[4] step is needed (same reasoning as v3 -> v4 above). v5 -> v6: added hint_suppression_probes (backoff-threshold probe-recovery counter for hint_stats.ts). Purely additive -- `CREATE TABLE IF NOT EXISTS` in SCHEMA_SQL already handles a pre-existing v5 database, so no MIGRATIONS[5] step is needed (same reasoning as v4 -> v5 above). v6 -> v7: added skill_version_snapshots (skill_version_drift.ts's one-shot "token-goat was upgraded since you loaded this skill" nudge). Purely additive -- `CREATE TABLE IF NOT EXISTS` in SCHEMA_SQL already handles a pre-existing v6 database, so no MIGRATIONS[6] step is needed (same reasoning as v5 -> v6 above). v7 -> v8: added notes (token-goat note-add/note-get/note-list -- file/symbol-attached architecture notes with a staleness fingerprint, see notes.ts). Purely additive -- `CREATE TABLE IF NOT EXISTS` in SCHEMA_SQL already handles a pre-existing v7 database, so no MIGRATIONS[7] step is needed (same reasoning as v6 -> v7 above). v8 -> v9: added symbols.parent, separating the "parent container name" the regex adapters used to overload into symbols.docstring from the symbol's real doc comment (see precedingDocComment in parser.ts and makeLineSymbol/makeSpanSymbol/makeSymbolEmitter in languages/common.ts). A pre-existing v8 database's `symbols` table predates the column, so it needs an explicit ALTER TABLE in MIGRATIONS -- not purely additive like v3 -> v8 above. v9 -> v10: added hint_emissions.bytes_emitted, the per-emission spend (byte length of the hint text actually injected into context) recorded alongside the pre-existing bytes-saved figures already tracked in the `stats` table, so `token-goat hint-stats` can answer "are hints net-positive" instead of only measuring their benefit (see applyHintTracking/logHintEmission in hint_stats.ts). Left nullable with no default rather than defaulted to 0, so a pre-existing v9 database's rows -- which predate spend tracking entirely -- read as genuinely unknown spend, not a fake measured zero. A pre-existing v9 database's `hint_emissions` table predates the column, so it needs an explicit ALTER TABLE in MIGRATIONS -- not purely additive like v3 -> v8 above. v11 -> v12: added embedding_provenance (which embedding stack produced the stored vectors, see the table's own comment and ensureEmbeddingProvenance in embeddings.ts). Purely additive -- `CREATE TABLE IF NOT EXISTS` in SCHEMA_SQL already handles a pre-existing v11 database, so no MIGRATIONS[11] step is needed. Deliberately no migration step even though existing vectors ARE invalidated by this change: a migration would have to run the invalidation on every database at open time, including ones whose embedding stack is absent and which therefore have nothing to re-embed with. The check belongs where embedding actually happens, and an empty provenance table on a populated index is the signal it keys on. v12 -> v13: added files.parser_sha, the digest of the extraction logic (src/parser.ts plus src/languages/**) that produced a file's rows, so a parser change invalidates already-indexed files whose content never moved. A pre-existing v12 database's `files` table predates the column, so it needs an explicit ALTER TABLE in MIGRATIONS -- not purely additive. v14 -> v15: added hint_emissions.observable, distinguishing a row whose verdict was actually observed from one that carried no pointer any later command could match, and backfilling it from `correlator IS NULL`. Not purely additive twice over: a pre-existing v14 table predates the column, and the backfill is a data change `CREATE TABLE IF NOT EXISTS` cannot express at all. v15 -> v16: added hint_emissions.displayed, so a detection that was suppressed before the agent ever saw it gets a row instead of vanishing -- the denominator of "how often did this fire" was previously unknowable from the ledger. A pre-existing v15 table predates the column, so it needs an explicit ALTER TABLE in MIGRATIONS; the DEFAULT covers every existing row without a backfill, since until v16 a row only existed when a hint was shown. v16 -> v17: added indexed_roots (projects indexed on purpose, see indexed_roots.ts), backfilled from every known_roots root that already has symbols, so a project indexed before the table existed keeps its indexed reminder and drift sweep. The table alone is additive; the backfill is the data change `CREATE TABLE IF NOT EXISTS` cannot express.
export const SCHEMA_VERSION = 17 as const
type Migration = (conn: SqliteDatabase) => void
/** Runs `sql` (expected to be an idempotent-in-intent `ALTER TABLE ... ADD COLUMN`), swallowing exactly a "duplicate column" failure -- the column already existing on a fresh database whose CREATE TABLE already includes it -- and rethrowing anything else, so a genuine ALTER TABLE failure is never silently lost. */
function alterTableIdempotent(conn: SqliteDatabase, sql: string): void {
try {
conn.exec(sql)
} catch (err) {
if (!(err instanceof Error) || !/duplicate column/i.test(err.message)) throw err
}
}
/** Delete every chunk (and matching vector) belonging to a dotenv file, and clear those files' `embed_sha` so they are re-embedded through the redacting path. Paths are filtered in JS with the same {@link isDotenvPath} predicate the redaction uses, rather than with a `LIKE '%.env%'` pattern, so this covers exactly the file set the fix covers and cannot drift from it. `chunk_vectors` is the optional sqlite-vec virtual table: on a build without the native extension the statement throws at prepare time, which is not a reason to fail the migration -- the chunk rows carrying the secret text are deleted either way, and a vector with no chunk row is unreadable (searchSemantic joins them by rowid). */
function purgeDotenvEmbeddings(conn: SqliteDatabase): void {
let paths: string[]
try {
paths = (conn.prepare('SELECT DISTINCT file_path FROM chunks').all() as { file_path: string }[])
.map((r) => r.file_path)
.filter(isDotenvPath)
} catch {
// No `chunks` table yet (a database created before embeddings existed): nothing to purge.
return
}
if (paths.length === 0) return
for (const p of paths) {
try {
conn.prepare('DELETE FROM chunk_vectors WHERE rowid IN (SELECT id FROM chunks WHERE file_path = ?)').run(p)
} catch {
// sqlite-vec not loaded on this build; the chunk rows below are still removed.
}
conn.prepare('DELETE FROM chunks WHERE file_path = ?').run(p)
try {
conn.prepare('UPDATE files SET embed_sha = NULL WHERE path = ?').run(p)
} catch {
// Older shape without the column; the v1 -> v2 step adds it, and a file with no embed_sha is never treated as fresh anyway.
}
}
}
/** Drop both FTS5 mirrors, re-create them from {@link FTS_SQL}, and repopulate each from its content table. Every other migration step here exists because `CREATE TABLE IF NOT EXISTS` cannot alter a populated table. This one exists because `CREATE VIRTUAL TABLE IF NOT EXISTS` cannot alter an EMPTY one either: against an existing table of that name it is a silent no-op, so a changed `tokenize=` argument would otherwise reach new databases only. Both tables are external-content (`content='symbols'` / `content='cache_recall'`), so the rows are not lost by the drop and `'rebuild'` re-derives the whole index from the base table. The triggers are left alone: they name the tables rather than depending on their declaration, and FTS_SQL's `CREATE TRIGGER IF NOT EXISTS` keeps them in place. A SQLite build without FTS5 has neither table, and every statement here throws at prepare time. That is not a migration failure -- higher layers already fall back to LIKE when the mirrors are absent (see the FTS_SQL call site in ensureSchema), and the same build could not have created them in the first place. Runs as one immediate transaction, which every previous step could do without. They add columns; this one drops a table that the `symbols` triggers write to, so between the DROP and the CREATE there is a window where an ordinary insert by another process fails with "no such table: main.symbols_fts" -- observed, not theorised: tests/index_concurrent_write_race.test.ts caught exactly that on the first run of this step, with one of eleven concurrently indexed files lost. Measured across six runs each way, that test fails 5 times in 6 without the transaction and 0 in 6 with it, so a single green run of it is not evidence this step is safe -- repeat it if this function changes. Taking the write lock up front instead makes that process wait out the rebuild under the 15s busy_timeout set in initConnection. `.immediate()` rather than the deferred default for the reason given on Database.transaction: a deferred transaction takes the lock at its first write, which here is already past the DROP. */
function rebuildFtsAtCurrentTokenizer(conn: SqliteDatabase): void {
try {
// Every migration step runs on a brand-new database too, which is stamped 0 and so walks the whole ladder. FTS_SQL created both tables at the current tokenizer moments earlier, so without this check a first-ever open would drop and re-create them for nothing -- and, less obviously, so would every test that opens a fresh index. Ask the database what it has rather than inferring it from the stamped version, which is also the honest check for a database whose tables were created by a build where FTS5 was missing and later restored.
const declarations = conn
.prepare("SELECT sql FROM sqlite_master WHERE name IN ('symbols_fts','cache_recall_fts')")
.all() as Array<{ sql: string | null }>
const current = declarations.map((d) => /tokenize\s*=\s*'([^']*)'/.exec(d.sql ?? '')?.[1] ?? '')
if (current.length === 2 && current.every((t) => t === FTS_TOKENIZER)) return
conn.transaction(() => {
conn.exec('DROP TABLE IF EXISTS symbols_fts; DROP TABLE IF EXISTS cache_recall_fts;')
conn.exec(FTS_SQL)
conn.exec("INSERT INTO symbols_fts(symbols_fts) VALUES('rebuild');")
conn.exec("INSERT INTO cache_recall_fts(cache_recall_fts) VALUES('rebuild');")
}).immediate()
} catch {
// FTS5 unavailable in this SQLite build -- search is already on the LIKE fallback path.
}
}
// Keyed by the FROM version: MIGRATIONS[1] upgrades a v1 database to v2, MIGRATIONS[2] upgrades v2 to v3, and so on. A version with no registered step is a no-op, which also covers a SCHEMA_VERSION bump for a purely additive change already handled by `CREATE TABLE IF NOT EXISTS` in SCHEMA_SQL (no ALTER TABLE needed).
const MIGRATIONS: Record<number, Migration> = {
// v1 -> v2: adds files.embed_sha, tracked separately from files.sha so embedding freshness can be gated independently of parse freshness (see makeIndexer in worker.ts). A pre-existing v1 database's `files` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has the column from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, so a genuine ALTER TABLE failure is never silently lost.
1: (conn) => alterTableIdempotent(conn, 'ALTER TABLE files ADD COLUMN embed_sha TEXT'),
// v2 -> v3: adds files.retry_count, a durable per-path counter for consecutive transient-read-failure requeues (see MAX_TRANSIENT_RETRIES / requeueDirtyPath / clearRetryCount in worker.ts). Previously this counter lived only in an in-memory Map inside worker.ts, which meant the retry-count reset -- run at the time in the short-lived hook CLI process -- could never actually reach the long-lived detached daemon process's own copy of that Map: they are different Node processes with no shared memory, so the reset was a silent no-op in the real deployed topology. Persisting the counter in `files` makes it visible to both processes via the one thing they do share: the index DB. Same swallow-duplicate-column pattern as v1 -> v2 above. Superseded since: the counter shares the DB but no longer the table, because holding it in `files` meant a never-indexed path got a `files` row the moment one read failed, and three readers take a `files` row to mean the file is indexed -- see the `index_retries` comment in SCHEMA_SQL above. This column is still created for databases that predate that move; nothing reads or writes it.
2: (conn) => alterTableIdempotent(conn, 'ALTER TABLE files ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0'),
// v8 -> v9: adds symbols.parent (see SCHEMA_VERSION comment above for why). A pre-existing v8 database's `symbols` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has the column from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 above.
8: (conn) => alterTableIdempotent(conn, "ALTER TABLE symbols ADD COLUMN parent TEXT NOT NULL DEFAULT ''"),
// v9 -> v10: adds hint_emissions.bytes_emitted (see SCHEMA_VERSION comment above for why). A pre-existing v9 database's `hint_emissions` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has the column from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 / v8 -> v9 above.
9: (conn) => alterTableIdempotent(conn, 'ALTER TABLE hint_emissions ADD COLUMN bytes_emitted INTEGER'),
// v10 -> v11: purge chunks (and their vectors) for dotenv files. Until this version, a tracked `.env` was chunked and embedded verbatim on the git path, so `semantic` returned its values -- see dotenv_redact.ts. Redacting from now on is not enough on its own: the embed-freshness gate (isEmbedFresh in parser.ts) skips a file whose bytes have not changed, so an already-indexed .env would have kept serving its pre-fix chunks indefinitely. Deleting the rows here both removes the stored secrets and, by clearing embed_sha, makes the next drain re-embed the file through the redacting path.
10: purgeDotenvEmbeddings,
// v12 -> v13: adds files.parser_sha, the digest of the extraction logic that produced this file's rows, tracked separately from files.sha for the same reason embed_sha is -- content freshness and parse freshness are different questions, and the content sha alone could only ever answer the first. A pre-existing v12 database's `files` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has it from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v1 -> v2 / v2 -> v3 / v8 -> v9 / v9 -> v10 above. Deliberately left NULL for every existing row rather than backfilled with the current fingerprint: NULL is the truthful answer (nobody recorded which parser wrote those rows), and it is also the answer that makes the freshness gates reparse them once, which is exactly what a database indexed by an older parser needs. v13 -> v14: changes both FTS5 tables' tokenizer to `unicode61 remove_diacritics 2`, so a search for `Noi` or `Viet` finds `Hà Nội` and `Việt Nam` -- combining marks that `remove_diacritics 1`, FTS5's default, leaves in place. This is the first schema change that `CREATE VIRTUAL TABLE IF NOT EXISTS` cannot express at all rather than merely cannot express on a populated table: against an existing virtual table that statement is a silent no-op, so without MIGRATIONS[13] the new tokenizer would reach only databases created after this release. The step drops both tables, re-runs FTS_SQL to re-create them at the current declaration, and rebuilds each from its content table.
12: (conn) => alterTableIdempotent(conn, 'ALTER TABLE files ADD COLUMN parser_sha TEXT'),
// v13 -> v14: re-creates both FTS5 tables at the tokenizer FTS_SQL currently declares (see the SCHEMA_VERSION comment above for why no `IF NOT EXISTS` form can do this).
13: rebuildFtsAtCurrentTokenizer,
// v14 -> v15: adds hint_emissions.observable and backfills it from the column that always implied it. A pre-existing v14 database's `hint_emissions` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has it from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v8 -> v9 / v9 -> v10 / v12 -> v13 above. The backfill is exact rather than a guess: until this version a NULL correlator was the only way a row could be born with nothing to wait for, so `correlator IS NULL` identifies precisely the population the new column names. It runs unconditionally after the ALTER because alterTableIdempotent swallows the duplicate-column case, and re-running the UPDATE on an already-migrated database sets the same rows to the same values.
14: (conn) => {
alterTableIdempotent(conn, 'ALTER TABLE hint_emissions ADD COLUMN observable INTEGER NOT NULL DEFAULT 1')
conn.prepare('UPDATE hint_emissions SET observable = 0 WHERE correlator IS NULL').run()
},
// v15 -> v16: adds hint_emissions.displayed (see the SCHEMA_VERSION comment above for why). A pre-existing v15 database's `hint_emissions` table predates the column, so it needs an explicit ALTER TABLE here; a brand-new database already has it from SCHEMA_SQL's CREATE TABLE above, so the ALTER TABLE would fail with "duplicate column name" there -- swallow exactly that error and rethrow anything else, same pattern as v8 -> v9 / v9 -> v10 / v12 -> v13 / v14 -> v15 above. No backfill, unlike v14 -> v15: until this version a row was only ever written when a hint was actually shown, so the column's own DEFAULT 1 is already the truth for every existing row, and an UPDATE would be a no-op dressed up as a migration step.
15: (conn) => alterTableIdempotent(conn, 'ALTER TABLE hint_emissions ADD COLUMN displayed INTEGER NOT NULL DEFAULT 1'),
// v16 -> v17: backfills indexed_roots.
16: backfillIndexedRoots,
}
/** Mark every known root that already has symbols as indexed. Until v17 "indexed" meant "has symbols", so this keeps that answer for every project a pre-v17 index already covered. known_roots is the only list of roots the index holds. A root whose only symbols came from one edited file is carried over too, which costs little: under the old rule its next session start queued the whole project anyway. */
function backfillIndexedRoots(conn: SqliteDatabase): void {
const scope = projectScopeClause('file_path')
const hasSymbols = conn.prepare(`SELECT 1 FROM symbols WHERE ${scope.clause} LIMIT 1`)
const insert = conn.prepare('INSERT OR IGNORE INTO indexed_roots (root, indexed_ms) VALUES (?, ?)')
const roots = conn.prepare('SELECT root, last_seen_ms FROM known_roots').all() as Array<{ root: string; last_seen_ms: number }>
for (const { root, last_seen_ms } of roots) {
if (hasSymbols.get(...scope.params(root)) !== undefined) insert.run(normalizePath(root), last_seen_ms)
}
}
// Walks a database from its stamped version up to (but not including) `toVersion`, applying each registered migration step in order. Does not itself touch PRAGMA user_version -- the caller stamps that once every step has run.
function runMigrations(conn: SqliteDatabase, fromVersion: number, toVersion: number): void {
for (let v = fromVersion; v < toVersion; v++) {
MIGRATIONS[v]?.(conn)
}
}
/** Apply pragmas + schema to a freshly opened connection. WAL journal mode allows a reader to proceed while a writer holds the file; NORMAL synchronous trades a small durability window for far fewer fsyncs, which matters on the hot hook path. FTS5 and the optional sqlite-vec table are best-effort: a SQLite build lacking either still yields a working DB. */
/** How long {@link enableWalWithRetry} keeps trying before giving up, matched to `busy_timeout`. */
const WAL_SWITCH_DEADLINE_MS = 15_000
/** Put a connection into WAL mode, waiting out other processes rather than failing on the first refusal. Converting a database's journal mode needs exclusive access, and SQLite answers `SQLITE_BUSY` for that conversion **without consulting the busy handler** -- the wait `busy_timeout` configures applies to ordinary lock contention, not to this. So on a database that does not exist yet, where every process racing to create it runs this conversion, `busy_timeout` cannot help and the losers throw immediately. Reproduced with six processes indexing one new database under load: one threw `database is locked` from this very pragma and dropped the file it was indexing, while the run still exited 0. That is the same silent-file-loss the deferred-`BEGIN` fix in `writeParseResult` removed, arriving by a second and entirely separate route -- which is why the comment that used to sit here, saying moving `busy_timeout` first was mere hardening because it "did not change it", was reporting a real remaining failure as a non-event. A process that loses the race has nothing to fix and nothing to report: whoever won is doing the conversion it wanted done. So each attempt re-reads the mode, and finding `wal` is success no matter who set it. Only a deadline passing with the database still not in WAL is an error, and it carries the last refusal so a genuine permission or filesystem problem is not reported as contention. `budgetMs` exists so the giving-up branch can be reached in a test without spending the real fifteen seconds to get there. Production callers pass nothing and get that full budget. `stopOn`, when given, ends the wait at the first refusal it matches. Only a caller that has somewhere else to go passes it: getDb with the read-only fallback on, which would otherwise spend the whole budget on a refusal no waiting cures (measured: 15.3 s per open against a write-denied index, three opens for one `semantic`). */
export function enableWalWithRetry(conn: Pick<SqliteDatabase, 'pragma'>, budgetMs: number = WAL_SWITCH_DEADLINE_MS, stopOn?: (e: unknown) => boolean): void {
const deadline = Date.now() + budgetMs
let lastError: unknown
for (;;) {
try {
const mode = conn.pragma('journal_mode = WAL', { simple: true })
if (String(mode).toLowerCase() === 'wal') return
lastError = new Error(`got: ${String(mode)}`)
} catch (e) {
lastError = e
if (isDamagedDatabaseError(e) || stopOn?.(e) === true) throw walSwitchError(e)
}
// Another process may already have finished the conversion while this one was being refused, in which case there is nothing left to do and no reason to keep waiting.
try {
if (String(conn.pragma('journal_mode', { simple: true })).toLowerCase() === 'wal') return
} catch {
// Reading the mode can fail for the same contention reason; fall through and retry.
}
if (Date.now() >= deadline) throw walSwitchError(lastError)
sleepSync(25)
}
}
/** Does `e` say the file is not a usable database at all? No other process finishes anything that clears that, so waiting out the deadline for it only delays the report: fifteen seconds for every open of a damaged global.db. */
function isDamagedDatabaseError(e: unknown): boolean {
const code = (e as { code?: unknown } | null)?.code
return typeof code === 'string' && /^SQLITE_(?:NOTADB|CORRUPT)/.test(code)
}
/** The error {@link enableWalWithRetry} gives up with. Carries the last refusal as `cause`, so {@link isWriteAccessError} can read its SQLite code through the wrapper. */
function walSwitchError(lastError: unknown): Error {
return new Error(`db: failed to enable WAL mode (${lastError instanceof Error ? lastError.message : String(lastError)})`, { cause: lastError })
}
/** Size a -wal file is cut back to once its log has been folded into the database and the next write restarts it. A WAL never shrinks on its own: SQLite reuses the file from its start and leaves it at the largest size it ever reached, and `journal_size_limit` is its only setting that truncates one, per connection and not stored in the database, so every read-write connection sets it. Without it one whole-database rewrite (a VACUUM in WAL mode writes every page through the log) whose truncating checkpoint was refused by a concurrent reader left a 5.0 GB -wal beside a 5.0 GB global.db for as long as any process held the database open, the hook server being one for days; all but 704 of its 1.2 million frames were stale. The limit is a floor for the file after a restart, not a cap: a transaction larger than it still writes what it must. */
export const WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024
/** The pragma that sets {@link WAL_SIZE_LIMIT_BYTES}. The setting belongs to the connection, so each read-write opener runs it: {@link initConnection} for the shared connection, and the per-call telemetry connection in stats.ts that opens the file without it. A string and not a function, so the pre-dispatch call graph that tests/guards/pre_handler_fs_touches_are_gated.test.ts counts gains no member. Read-only and in-memory connections never write a log and skip it. */
export const WAL_SIZE_LIMIT_PRAGMA = `journal_size_limit = ${WAL_SIZE_LIMIT_BYTES}`
/** How long a connection opened by a one-shot hook process waits on another process's lock, and how long its WAL switch is retried, in milliseconds. The hook is the agent's tool call: every wait here is wall-clock the agent sits through, and every write a hook makes is a hint, a stat or a session note that is fail-soft by contract, so a lost row costs far less than the 15 s a wedged lock holder charged per write (measured 32.8-38.3 s for one Read whose hint made two ledger writes). Sized like {@link PROBE_BUSY_TIMEOUT_MS}, the other pre-tool read bound. */
const HOOK_DB_WAIT_MS = 250
let _hookWaits = false
/** Set the hook/patient busy timeout on all cached connections and select that budget for future connections; `on = false` restores the patient budget. */
export function limitDbWaitsForHook(on = true): void {
_hookWaits = on
_connections.forEach((conn) => conn.pragma(`busy_timeout = ${connectionWaitBudgets().busyMs}`))
}
/** The busy_timeout and the WAL-switch deadline, in milliseconds, that a connection opened now gets: the patient indexing budgets, or the hook's short ones after {@link limitDbWaitsForHook}. */
export function connectionWaitBudgets(): { busyMs: number; walSwitchMs: number } {
return _hookWaits ? { busyMs: HOOK_DB_WAIT_MS, walSwitchMs: HOOK_DB_WAIT_MS } : { busyMs: 15000, walSwitchMs: WAL_SWITCH_DEADLINE_MS }
}
function initConnection(conn: SqliteDatabase, walStopOn?: (e: unknown) => boolean): void {
// busy_timeout makes a writer wait for a held write lock instead of failing immediately with SQLITE_BUSY; token-goat runs multiple processes against one global.db (worker daemon draining the queue plus CLI hook invocations), so concurrent writers are normal and 15s absorbs contention spikes without hanging (15s is the budget of a worker or CLI process; a hook process gets the short one from connectionWaitBudgets). Set FIRST, before any statement that can contend, rather than after the two pragmas below as it used to be. The switch to WAL and the schema creation that follows it both need an exclusive lock, and on a database that does not exist yet every process racing to create it runs both -- with the timeout armed only afterwards, those two steps ran with SQLite's default of no wait at all. This is hardening rather than a fix for a reproduced failure: the concurrent-index failure that prompted the look was a deferred-BEGIN upgrade elsewhere (see writeParseResult in parser.ts), and moving this line did not change it.
const waits = connectionWaitBudgets()
conn.pragma(`busy_timeout = ${waits.busyMs}`)
enableWalWithRetry(conn, waits.walSwitchMs, walStopOn)
conn.pragma('synchronous = NORMAL')
conn.pragma(WAL_SIZE_LIMIT_PRAGMA)
conn.pragma('cache_size = -32000')
conn.pragma('temp_store = MEMORY')
conn.pragma('mmap_size = 134217728')
registerTgLower(conn)
// A single cheap read on every open -- this runs on the hot path (every hook call, every CLI invocation), so no schema work happens here beyond one PRAGMA read. Anything ABOVE SCHEMA_VERSION means an older binary opened a database written by a newer one (a downgrade, or two globally-installed versions pointed at the same project): refuse rather than risk an old binary misinterpreting or corrupting a schema shape it doesn't understand.
const storedVersion = Number(conn.pragma('user_version', { simple: true }))
if (storedVersion > SCHEMA_VERSION) throw newerSchemaError(storedVersion)
conn.exec(SCHEMA_SQL)
try {
conn.exec(FTS_SQL)
} catch {
// FTS5 unavailable in this SQLite build — search falls back to LIKE in higher layers. The base tables are still usable.
}
// sqlite-vec is an optional dependency; the vec0 virtual table only exists when the package is installed and its extension can be loaded. A missing package, a load failure or a failed create is non-fatal.
if (loadSqliteVec(conn)) {
try {
conn.exec(
`CREATE VIRTUAL TABLE IF NOT EXISTS chunk_vectors USING vec0(
embedding float[384]
);`,
)
} catch {
// The extension loaded but the table could not be created: semantic search is disabled but every other index feature works.
}
}
// BELOW SCHEMA_VERSION covers two cases identically: a brand-new DB (storedVersion 0, tables just created above) and an old DB from a pre-migration-mechanism release (also storedVersion 0, since older code never stamped it, but already schema-shape-compatible with version 1 -- that constant IS today's schema). Both are stamped current with no real migration step to run. A genuine future gap -- SCHEMA_VERSION bumped for a change `CREATE TABLE IF NOT EXISTS` can't express, e.g. an ALTER TABLE on an existing table -- runs through MIGRATIONS above.
if (storedVersion < SCHEMA_VERSION) {
runMigrations(conn, storedVersion, SCHEMA_VERSION)
conn.pragma(`user_version = ${SCHEMA_VERSION}`)
}
}
/** The refusal for a database stamped by a newer build than this one, shared by the read-write and read-only opens. */
function newerSchemaError(storedVersion: number): Error {
return new Error(
`db: index schema version ${storedVersion} is newer than this token-goat build supports (expected ${SCHEMA_VERSION}). ` +
`Update token-goat, or delete the stale index database and let it rebuild.`,
)
}
/** Custom Unicode-aware LOWER() replacement used by pathEqClause() (sql_path.ts) for case-insensitive-filesystem path comparisons. SQLite's built-in LOWER() only folds ASCII A-Z, which would silently diverge from foldPath()'s JS-side Unicode-aware toLowerCase() for non-ASCII casing (e.g. `Ä` vs `ä`). Wrapping the exact same foldCase() primitive here keeps SQL-side and JS-side folding byte-for-byte consistent. Registered once per connection (not per-query) and marked deterministic so SQLite can use it in query planning the same way it would a built-in function. */
function registerTgLower(conn: SqliteDatabase): void {
conn.function('TG_LOWER', { deterministic: true }, (value: unknown) =>
value === null ? null : foldCase(String(value)),
)
}
/** Load the optional sqlite-vec extension into `conn`; false when the package is missing or its extension fails to load. Required dynamically so a missing package does not break module resolution. */
function loadSqliteVec(conn: SqliteDatabase): boolean {
try {
const sqliteVec = _require('sqlite-vec') as { load: (db: SqliteDatabase) => void }
sqliteVec.load(conn)
return true
} catch {
return false
}
}
/** Set by {@link allowReadOnlyIndex}: when present, getDb may answer from a read-only connection to a database it cannot open for writing, and calls this once, the first time it does. */
let _readOnlyFallback: { onFirstUse: () => void; used: boolean } | undefined
/** Read-only connections getDb opened because the read-write open was refused, keyed like {@link _connections}. Kept apart from that map so a read-only handle is only ever returned while {@link allowReadOnlyIndex} is in force: a writer never opts in, so it never receives one. */
const _readOnlyConnections = new Map<string, SqliteDatabase>()
/** Let getDb answer from a read-only connection when a database cannot be opened for writing: permission denied, a read-only filesystem or sandbox, or a WAL switch refused for that reason. Only a process that does nothing but query the index may call this, and only before its first getDb: the CLI calls it for the commands in cli.ts's READ_ONLY_INDEX_COMMANDS. The worker, `index`, `doctor`, the hooks and every other writer never do, so they keep failing loudly on the same database. Within an opted-in process a write through the read-only handle throws SQLITE_READONLY rather than succeeding somewhere else; the read path's own writes (stats, stale-file reindexing) check {@link isReadOnlyDb} and stand down. `onFirstUse` runs once, when the first read-only connection is opened, so the caller can say that this run neither records stats nor reindexes. */
export function allowReadOnlyIndex(onFirstUse: () => void): void {
_readOnlyFallback = { onFirstUse, used: false }
}
/** True when getDb is serving `dbPath` through a read-only fallback connection. In a process that called {@link allowReadOnlyIndex} this opens the connection if nothing has yet, because only that open decides whether the run can write: `--force-refresh` asks before any query has run. */
export function isReadOnlyDb(dbPath: string): boolean {
// The opt-in check comes first: every read-path caller asks this on each call, and without it no read-only connection can exist, while connectionKey resolves the path on disk (and refuses a `:memory:` path outright).
if (_readOnlyFallback === undefined) return false
try {
getDb(dbPath, { create: false })
} catch {
// Not read-only, just unopenable: the caller's own getDb reports why.
return false
}
return _readOnlyConnections.has(connectionKey(dbPath).key)
}
const WRITE_ACCESS_SQLITE_CODE_RE = /^SQLITE_(?:CANTOPEN|READONLY|PERM|IOERR_ACCESS|IOERR_SHMOPEN)/
const WRITE_ACCESS_FS_CODES = new Set(['EACCES', 'EPERM', 'EROFS'])
/** Does `e` say this process may not write the database or create its -wal/-shm files, as opposed to contention, corruption or a schema it cannot read? Follows `cause`, which is how {@link walSwitchError} carries SQLite's code. */
function isWriteAccessError(e: unknown): boolean {
for (let cur: unknown = e, depth = 0; cur instanceof Error && depth < 4; cur = cur.cause, depth++) {
const code = (cur as { code?: unknown }).code
if (typeof code === 'string' && (WRITE_ACCESS_SQLITE_CODE_RE.test(code) || WRITE_ACCESS_FS_CODES.has(code))) return true
}
return false
}
/** The error opening `resolved` for writing raises when the file exists and this process may not write it, else undefined. SQLite cannot be asked: its read-write open of a write-protected file quietly becomes a read-only one, so an index whose -wal and -shm already exist opens, answers, and fails only at its first write, with nothing said about why. */
function writeRefusal(resolved: string): unknown {
let fd: number
try {
fd = fs.openSync(resolved, 'r+')
} catch (e) {
return isWriteAccessError(e) ? e : undefined
}
fs.closeSync(fd)
return undefined
}
/** Open `resolved` read-only and read its schema version, closing the handle if the read fails. `busyTimeoutMs` is how long a read waits on a writer's lock. */
function openReadOnlyAt(resolved: string, immutable: boolean, busyTimeoutMs = 15000): { conn: SqliteDatabase; storedVersion: number } {
const conn = new Database(resolved, { readonly: true, fileMustExist: true, immutable })
try {
conn.pragma(`busy_timeout = ${busyTimeoutMs}`)
return { conn, storedVersion: Number(conn.pragma('user_version', { simple: true })) }
} catch (e) {
try {
conn.close()
} catch {
// Best-effort: the read failure is what the caller acts on.
}
throw e
}
}
/** Open an existing index read-only: no WAL switch, no schema DDL, no migrations, no sqlite-vec table creation, only the TG_LOWER function and the sqlite-vec module the queries need. A plain read-only open of a WAL database still needs its -wal and -shm files to exist or be creatable. A clean close deletes both, so on a write-denied directory that open fails (measured on Windows: "unable to open database file"), and the second attempt opens with `immutable=1`, which reads the main file alone. That trades two things away: SQLite then ignores whatever the -wal holds, so the answer is as of the last checkpoint, and it takes no locks, so a writer elsewhere that checkpoints mid-query can make that query fail. Neither can change the file, which this process has no way to write. A stored schema version older than SCHEMA_VERSION is refused rather than served. Migrations cannot run without write access, and the ones they would have run matter to a reader: v10 -> v11 purges `.env` values that older builds embedded verbatim, which `semantic` would otherwise return, v13 -> v14 rebuilds the FTS tables for the current tokenizer, and several steps add columns the queries read. A newer version is refused with the same error the read-write open gives. */
function openIndexReadOnly(resolved: string): SqliteDatabase {
let opened: { conn: SqliteDatabase; storedVersion: number }
try {
opened = openReadOnlyAt(resolved, false)
} catch (e) {
if (!isWriteAccessError(e)) throw e
opened = openReadOnlyAt(resolved, true)
}
const { conn, storedVersion } = opened
try {
if (storedVersion > SCHEMA_VERSION) throw newerSchemaError(storedVersion)
if (storedVersion < SCHEMA_VERSION) {
throw new Error(
`db: index schema version ${storedVersion} predates this token-goat build (expected ${SCHEMA_VERSION}) and cannot be upgraded without write access. ` +
`Run token-goat once where its data directory is writable.`,
)
}
registerTgLower(conn)
loadSqliteVec(conn)
} catch (e) {
try {
conn.close()
} catch {
// Best-effort: the version error is what the caller needs.
}
throw e
}
return conn
}
/** Resolve a db path argument to an absolute path under the data directory. A bare filename (no directory separator) is placed in {@link dataDir}; an already-absolute or explicitly-relative path is resolved as given so callers can point at a temp file in tests. */
function resolveDbPath(dbPath: string): string {
if (path.isAbsolute(dbPath)) return dbPath
if (dbPath.includes('/') || dbPath.includes('\\')) return path.resolve(dbPath)
return safeJoin(dataDir(), dbPath)
}
/** Resolves `dbPath` and derives its connection-cache key in one step -- shared by getDb and closeDb, which both need the same resolved-path-then-fold sequence to address the same cached handle. */
function connectionKey(dbPath: string): { resolved: string; key: string } {
const resolved = resolveDbPath(dbPath)
return { resolved, key: foldPath(resolved) }
}
/** How long the pre-read probe waits on a writer, in milliseconds, before answering nothing. */
const PROBE_BUSY_TIMEOUT_MS = 250
/** Run a short, bounded read-only callback on the index database if it exists and is ready. Never runs migrations or DDL, waits at most {@link PROBE_BUSY_TIMEOUT_MS} on a writer, and catches all errors (returning null). A data directory this process may not write is read through the same immutable retry {@link openIndexReadOnly} makes, so the probe still answers there. A schema version other than this build's is null, not an error. Ensures fail-open zero-contention behavior for pre-tool-use hooks. */
export function withProbeIndex<T>(
fn: (db: SqliteDatabase) => T,
dbPath?: string,
): T | null {
const resolved = resolveDbPath(dbPath ?? globalDbPath())
if (!fs.existsSync(resolved)) return null
let conn: SqliteDatabase | null = null
try {
let opened: { conn: SqliteDatabase; storedVersion: number }
try {
opened = openReadOnlyAt(resolved, false, PROBE_BUSY_TIMEOUT_MS)
} catch (e) {
if (!isWriteAccessError(e)) throw e
opened = openReadOnlyAt(resolved, true, PROBE_BUSY_TIMEOUT_MS)
}
conn = opened.conn
if (opened.storedVersion !== SCHEMA_VERSION) return null
registerTgLower(conn)
return fn(conn)
} catch {
return null
} finally {
if (conn) {
try {
conn.close()
} catch {
// best-effort cleanup
}
}
}
}
// What a read-only caller sees in place of a database file that does not exist yet: the index tables, empty. Shared, never written, closed with the rest.
let _emptyIndex: SqliteDatabase | undefined
/** {@link getDb} for a caller that only reads: when the file does not exist yet it answers from an empty index instead of creating the file, so a query on a fresh data directory finds nothing and leaves nothing behind (an empty `global.db` would also make `doctor` report an index that was never built). Writers keep using {@link getDb}. */
export function getReadDb(dbPath: string): SqliteDatabase {
return getDb(dbPath, { create: false })
}
/** Return the cached {@link SqliteDatabase} for `dbPath`, opening and initializing it on first access. The connection is opened with the schema applied, WAL enabled, and the optional FTS5 / sqlite-vec tables created when available. Subsequent calls with the same resolved path return the same handle. In a process that called {@link allowReadOnlyIndex}, a database it may not write is served through a read-only connection instead (see {@link openIndexReadOnly}); in every other process that refusal propagates. */
export function getDb(dbPath: string, opts: { create?: boolean } = {}): SqliteDatabase {
// Fold only the cache key, not `resolved` itself -- the real-case path is still what gets passed to fs/Database below, so the file is created/opened with whatever casing the caller (or an existing file on disk) actually used.
const { resolved, key } = connectionKey(dbPath)
const existing = _connections.get(key)
if (existing !== undefined) return existing
if (opts.create === false && !fs.existsSync(resolved)) {
if (_emptyIndex === undefined) {
_emptyIndex = new Database(':memory:')
registerTgLower(_emptyIndex)
_emptyIndex.exec(SCHEMA_SQL)
try {
_emptyIndex.exec(FTS_SQL)
} catch {
// FTS5 unavailable, as in initConnection: readers fall back to LIKE.
}
}
return _emptyIndex
}
const fallback = _readOnlyFallback
if (fallback !== undefined) {
const readOnly = _readOnlyConnections.get(key)
if (readOnly !== undefined) return readOnly
const refused = writeRefusal(resolved)
if (refused !== undefined) return openReadOnlyFallback(resolved, key, fallback, refused)
}
// Ensure the parent directory exists before SQLite tries to create the file. Retry on Windows race conditions.
const dir = path.dirname(resolved)
try {
ensureDirSync(dir)
} catch (e) {
if ((e as NodeJS.ErrnoException).code !== 'EEXIST' || !fs.existsSync(dir)) throw e
}
let conn: SqliteDatabase | undefined
try {
conn = new Database(resolved)
initConnection(conn, fallback !== undefined ? isWriteAccessError : undefined)
} catch (e) {
// A setup step (WAL pragma, schema exec, ...) failed after the handle was already opened. Close it before propagating so the failure does not leak a file descriptor.
try {
conn?.close()
} catch {
// Best-effort: the original setup error is what matters to the caller.
}
if (fallback !== undefined && isWriteAccessError(e)) return openReadOnlyFallback(resolved, key, fallback, e)
throw e
}
_connections.set(key, conn)
return conn
}
/** The read-only half of getDb, reached only with {@link allowReadOnlyIndex} in force and a read-write open refused for lack of write access. When the read-only open fails too, the read-write refusal is what the caller sees, unless the database was readable and turned out to be one this build cannot serve (a schema version error), which is the more useful thing to say. */
function openReadOnlyFallback(resolved: string, key: string, fallback: { onFirstUse: () => void; used: boolean }, writeError: unknown): SqliteDatabase {
let conn: SqliteDatabase
try {
conn = openIndexReadOnly(resolved)
} catch (e) {
throw isWriteAccessError(e) || !fs.existsSync(resolved) ? writeError : e
}
_readOnlyConnections.set(key, conn)
if (!fallback.used) {
fallback.used = true
fallback.onFirstUse()
}
return conn
}
/** Close the cached connection for `dbPath` if one is open. No-op otherwise. */
export function closeDb(dbPath: string): void {
const { key } = connectionKey(dbPath)
for (const cache of [_connections, _readOnlyConnections]) {
const conn = cache.get(key)
if (conn === undefined) continue
try {
conn.close()
} catch {
// Already closed or close raced with another caller — the handle is gone either way, so dropping it from the map is the only thing that matters.
}
cache.delete(key)
}
}
/** Close every open connection and clear the cache. Registered with {@link registerReset} so tests start from a clean slate, and usable directly for process shutdown. */
export function closeAllDbs(): void {
closeAllDbsCollect()
}
/** As {@link closeAllDbs}, which discards close errors, but returns them: every handle is still attempted and the cache cleared, so a caller that must not lose a failure (a test fixture's teardown) can report it after the rest of its own cleanup ran. */
export function closeAllDbsCollect(): unknown[] {
const errors: unknown[] = []
try {
_emptyIndex?.close()
} catch (err) {
errors.push(err)
}
_emptyIndex = undefined
for (const cache of [_connections, _readOnlyConnections]) {
for (const conn of cache.values()) {
try {
conn.close()
} catch (err) {
errors.push(err)
}
}
cache.clear()
}
return errors
}
// Per request too: a resident hook server that kept a handle open would pin the file against `--purge` and, on Windows, against deletion, and would keep writing to a database replaced underneath it.
registerReset(closeAllDbs, { perRequest: true })
registerReset(() => {
_readOnlyFallback = undefined
}, { perRequest: true })
// Per request too: the resident hook server switches to the hook waits for a hook request and must hand the patient ones back to the CLI requests it serves next.
registerReset(() => {
_hookWaits = false
}, { perRequest: true })