Repository navigation
Expand file tree
/
Copy pathparser.ts
More file actions
969 lines (875 loc) · 81.7 KB
/
Copy pathparser.ts
File metadata and controls
969 lines (875 loc) · 81.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
/** Tree-sitter source indexer (Layer 7). Parses a source file into {@link SymbolEntry} / {@link RefEntry} rows and upserts them into the SQLite index that `index_reader.ts` and the CLI read back. Tree-sitter grammars (TypeScript / JavaScript / Python) are optional native dependencies: when a grammar fails to load — a build without the native binding, or an unsupported language — extraction degrades to a regex pass that still recovers top-level functions and classes. The Python port (`parser.py`) keeps a richer model (imports/exports, sections, per-file SHA gating). This TS port targets the symbol/ref subset that the read commands surface, matching the simplified `db.ts` schema whose `symbols`/`refs` rows are keyed by absolute `file_path`. */
import * as fs from 'node:fs'
import { createRequire } from 'node:module'
import * as path from 'node:path'
import { isNonTextAsset, NON_TEXT_ASSET_SET_ID } from './asset_extensions.js'
import { globalDbPath, SYMBOL_BODY_CHAR_CAP } from './constants.js'
import { getDb } from './db.js'
import type { SqliteDatabase } from './sqlite_driver.js'
import { loadConfig } from './config.js'
import type { IndexingConfig } from './config.js'
import { redactIfDotenv } from './dotenv_redact.js'
import { chunkFile, deleteFileEmbeddings, indexFile as embedIndexFile } from './embeddings.js'
import { buildEmbeddingBoundaries } from './embedding_boundaries.js'
import { isEmbeddableDocument, extractEmbeddableDocumentText, isDocumentRefusal, isTransientDocumentRefusal } from './doc_embed_extract.js'
import { MAX_DOCUMENT_WORK_MILLIS } from './document_refusal.js'
import { fingerprintContent } from './fingerprint.js'
import { pathEqClause } from './sql_path.js'
import { hasSkipSegmentBelowRoot } from './skip_scope.js'
import { normalizeSourceText } from './source_text.js'
import { parserFingerprintForLanguage } from './parser_stamp.js'
import { detectLanguage, refineLanguageByContent, TREE_SITTER_LANGUAGES } from './parser_types.js'
import type { Language, RefEntry, SymbolEntry } from './parser_types.js'
import type { RegexLanguage } from './language_specs.js'
import type * as RegexAdapters from './languages/registry.js'
import type * as SalesforceFrontend from './languages/salesforce_frontend.js'
import { ipynbToVirtualSource } from './languages/ipynb_idx.js'
import { decodeSource, foldPath, isCaseInsensitiveFs } from './util.js'
import { normalizePath } from './paths.js'
const _require = createRequire(import.meta.url)
/** Result of parsing one file: extracted symbols, refs, language, timing. */
export interface ParseResult {
readonly symbols: SymbolEntry[]
readonly refs: RefEntry[]
readonly language: Language
readonly duration: number
}
/** Ref names that are never hand-written user code, so a ref bearing one carries no retrieval value under any query and is dropped before it reaches the `refs` table (see the insert loop in {@link writeParseResult}). This is deliberately a very small, unambiguous set, not a general "assertion noise" stoplist -- see the S3 finding this closes for the fuller reasoning: `__name` is esbuild's own compiler-injected helper (it rewrites `function foo(){}` into `foo = __name(function(){...}, "foo")` for `Function.prototype.name` support in its bundle output). Confirmed against a real indexed corpus: every `__name` ref traced back to a `*.snapshot/asset.*.bundle/index.js` file -- a vendored, pre-bundled build artifact, not source a developer wrote. No language grammar this indexer parses ever produces `__name` as a user-authored call site, so excluding it carries no risk of dropping a real caller. Deliberately NOT included here: generic test-framework globals (`expect`, `it`, `describe`, `toBe`, ...), even though they dominate refs-table row count in a JS/TS-heavy index. A project can legitimately define its own same-named export (a custom `describe`/`it`-shaped DSL is not far-fetched), and silently dropping those refs at index time would make `callers describe` permanently return zero for such a project with no error -- the "silently-emptied enumeration passes forever" failure class this codebase has shipped and fixed before. That risk, plus the relatively narrow benefit (each of those names is opt-in: nobody queries `callers expect` by accident), is why S3 did not add a stoplist for them. */
const COMPILER_ARTIFACT_REF_NAMES: ReadonlySet<string> = new Set(['__name'])
// --- Tree-sitter grammar loading (optional, cached) -------------------------
// Minimal structural typings for the node-tree-sitter API surface we touch. The packages ship no first-class .d.ts under this resolution, so we model only the members used here rather than pulling `any` through the module.
import type { Grammar, TsParser, TsParserCtor } from './parser_ts_types.js'
export type { Grammar, TsNode, TsParser, TsParserCtor, TsPoint, TsTree } from './parser_ts_types.js'
import {
extractCppSymbols,
extractGoSymbols,
extractJavaSymbols,
extractPythonSymbols,
extractRubySymbols,
extractRustSymbols,
extractTsJsSymbols,
} from './parser_treesitter.js'
export {
extractCppSymbols,
extractGoSymbols,
extractJavaSymbols,
extractPythonSymbols,
extractRubySymbols,
extractRustSymbols,
extractTsJsSymbols,
stripPythonStringQuotes,
} from './parser_treesitter.js'
import { extractRefs } from './parser_refs.js'
export { extractRefs } from './parser_refs.js'
export const REF_LANGUAGES: ReadonlySet<Language> = new Set<Language>([
'typescript',
'javascript',
'python',
'go',
'rust',
'java',
'c',
'cpp',
'ruby',
'csharp',
])
import {
extractCssSymbols,
extractDockerfileSymbols,
extractJsonSymbols,
extractMarkdownSymbols,
extractTomlSymbols,
extractWithRegex,
extractYamlSymbols,
} from './parser_structured.js'
export {
extractCssSymbols,
extractDockerfileSymbols,
extractJsonSymbols,
extractMarkdownSymbols,
extractTomlSymbols,
extractWithRegex,
extractYamlSymbols,
lineOpenDelimiterAfter,
stripTomlComment,
tomlBracketDelta,
yamlLineClosesQuote,
yamlOpenQuoteAfter,
} from './parser_structured.js'
// Cache the Parser constructor and each resolved grammar across calls so the native binding is loaded at most once per process. `null` means "tried and unavailable"; `undefined` means "not yet attempted". Keyed by string, not just `Language`, because TypeScript has two grammar variants (plain `.ts` vs JSX-aware `.tsx`) sharing one `Language` value.
let _parserCtor: TsParserCtor | null | undefined
// Last error from loading the core `tree-sitter` binding, kept for `token-goat doctor` diagnostics.
let _parserCtorError: Error | null = null
// Test-only override for the core binding: `undefined` means "use the real lazy-loaded module", `null` or a constructor forces that value instead.
let _parserCtorOverride: TsParserCtor | null | undefined = undefined
// Test-only load error reported alongside a forced binding, so doctor's classification runs on a real captured error.
let _parserCtorErrorOverride: Error | null | undefined = undefined
const _grammarCache = new Map<string, Grammar | null>()
// Tree-sitter parsing is synchronous, so one native parser per resolved grammar can be reused safely on the current thread.
const _parserByGrammar = new Map<Grammar, TsParser>()
function loadParserCtor(): TsParserCtor | null {
if (_parserCtorOverride !== undefined) return _parserCtorOverride
if (_parserCtor !== undefined) return _parserCtor
try {
_parserCtor = _require('tree-sitter') as TsParserCtor
} catch (e) {
_parserCtor = null
_parserCtorError = e instanceof Error ? e : new Error(String(e))
}
return _parserCtor
}
/** True when the core `tree-sitter` native binding is installed and requires cleanly. Independent of any single grammar (see `isTreeSitterAvailable`), since a missing core binding disables every language at once. */
export function treeSitterCoreAvailable(): boolean {
return loadParserCtor() !== null
}
/** Last error from loading the core `tree-sitter` binding, for diagnostics (`token-goat doctor` style callers). `null` when never attempted, attempted successfully, or overridden for testing. */
export function treeSitterCoreLoadError(): Error | null {
return _parserCtorErrorOverride !== undefined ? _parserCtorErrorOverride : _parserCtorError
}
/** Test-only: force `loadParserCtor()` (and therefore `treeSitterCoreAvailable()`/`isTreeSitterAvailable()`) to use `mod` instead of the real lazy-loaded binding. Pass `undefined` to restore the real resolution. */
export function setTreeSitterCoreForTesting(mod: TsParserCtor | null | undefined, loadError: Error | null = null): void {
_parserCtorOverride = mod
_parserCtorErrorOverride = mod === undefined ? undefined : loadError
_parserByGrammar.clear()
}
// `.h` is inherently ambiguous between C and C++ (unlike `.hpp`, which is unambiguous cpp) -- the same extension is used for both languages. None of `class`, `namespace`, `template<`, `::`, or an access-specifier label (`public:`/`private:`/`protected:`) are valid C syntax, so any match is a strong, low-false-positive signal that a `.h` file is genuinely a C++ header. Used to route such files to the cpp grammar instead of the c grammar -- parsing C++-only syntax with tree-sitter-c still "succeeds" (error recovery doesn't throw) but produces ERROR nodes around it, silently dropping or mis-scoping symbols/refs, exactly the same failure mode `useTsx` below guards against.
const CPP_HEADER_SNIFF_RE = /\bclass\s+\w|\bnamespace\s+\w|\btemplate\s*<|::\s*\w|\b(?:public|private|protected)\s*:/
/** Hard ceiling on the source text stored in `symbols.body`, in characters. `symbols.body` is written once per symbol and mirrored again into the `symbols_fts` full-text index, so an extractor that emits an oversized body costs roughly double on disk and is tokenized by FTS on every reindex. Any extractor bug that makes body size scale with *file* size rather than with *symbol* size therefore inflates the index quadratically, which is what {@link extractJsonSymbols} used to do for minified JSON (see the comment there). A DB bloated that way makes a reindex of the affected file long enough to hold SQLite's single writer lock past db.ts's 15s `busy_timeout`, which surfaces to a concurrent writer (worker daemon vs. CLI/hook) as "database is locked", and stalls `token-goat index` while FTS re-tokenizes gigabytes of duplicated text. This cap is enforced at the single write path ({@link writeParseResult}) rather than in each extractor, so it bounds every current and future language extractor, not just the one that regressed. 128 KB is far above any realistic single function/class body (~32k tokens) while still bounding the pathological case. */
export const MAX_SYMBOL_BODY_CHARS = SYMBOL_BODY_CHAR_CAP
/** Bound what gets *stored* for a symbol body, without losing what gets *read*. An over-cap body is stored as the empty string rather than as a truncated copy. That distinction is the whole point: read_commands.ts's `resolveBody` — the shared accessor behind `read`, `symbol`, `brief`, and frame resolution — already re-slices an empty body from the source file using the symbol's line range, so an elided body is served back complete and correct from disk. Storing a *truncated* body instead would defeat that fallback and make every one of those commands silently return partial source while presenting it as the full symbol, with `line_end` still advertising the complete range. The cost of eliding is confined to what genuinely needs the text resident in the DB: FTS body matching for that one oversized symbol. Losing full-text hits on a >128 KB body is a fair trade against unbounded index growth, and against `read` lying about what it returned. */
export function boundSymbolBody(body: string): string {
return body.length > MAX_SYMBOL_BODY_CHARS ? '' : body
}
/** Ceiling on a stored `symbols.docstring`. Smaller than the body cap because a docstring is a summary: past a few KB it has stopped being one, and nothing displays it in full. */
export const MAX_SYMBOL_DOCSTRING_CHARS = 16 * 1024
/** Marker appended to a docstring cut at {@link MAX_SYMBOL_DOCSTRING_CHARS}. */
const DOCSTRING_TRUNCATION_MARKER = '\n[... docstring truncated by token-goat ...]'
/** Bound a stored docstring, **truncating** rather than eliding it. This is deliberately the opposite of {@link boundSymbolBody}, for a reason specific to the column. An over-cap *body* is stored empty because read_commands.ts's resolveBody can rebuild it exactly from `[line_start, line_end]` -- eliding costs nothing. No such range is recorded for a docstring, so eliding one destroys it: `outline`'s documented/undocumented flag would flip to "undocumented" for the most heavily documented symbols in a file, which is worse than a visibly-cut docstring. Nothing treats this column as complete source the way `read` treats `body` -- consumers display its first line (read_commands.ts), test it for emptiness, or split it into words (graph_commands.ts) -- so a marked truncation is honest and lossless enough. Python's extractor (`pythonDocstring`) and {@link precedingDocComment}'s callers (TS/JS, Rust, Go, Ruby, Java, C/C++, and the regex fallback) are the only ones that populate `docstring` -- every other extractor still sets `''`. The column is derived from a *region near* a symbol, which is precisely the shared-region shape that made `body` grow quadratically: a file-level doc comment attributed to every symbol in the file reproduces that bug exactly. Bounding it at the same choke point closes the hole before a future extractor opens it. */
export function boundSymbolDocstring(docstring: string): string {
if (docstring.length <= MAX_SYMBOL_DOCSTRING_CHARS) return docstring
// Budget the marker inside the cap, not on top of it: appending it to a full-length slice would make the stored value exceed the very bound this constant declares, which quietly turns a hard storage limit into an approximate one.
let end = MAX_SYMBOL_DOCSTRING_CHARS - DOCSTRING_TRUNCATION_MARKER.length
// Never cut between the halves of a surrogate pair -- that stores a lone surrogate, which is not valid text and can surface as a replacement character or upset consumers downstream.
const cutsSurrogatePair =
end > 0 && docstring.charCodeAt(end - 1) >= 0xd800 && docstring.charCodeAt(end - 1) <= 0xdbff
if (cutsSurrogatePair) end -= 1
return docstring.slice(0, Math.max(0, end)) + DOCSTRING_TRUNCATION_MARKER
}
/** Ceiling on a stored `refs.context`. A ref's context is one display line -- every consumer renders it as `path:line: context` -- so anything past a couple of KB is already unreadable where it is shown. */
export const MAX_REF_CONTEXT_CHARS = 2048
/** Marker standing in for the part of a context line cut at {@link MAX_REF_CONTEXT_CHARS}. */
const REF_CONTEXT_ELISION = ' […] '
/** Bound a stored ref context to a window around the reference itself, rather than to the head of its line. A "line" is only short in a file a human wrote: a generated FlexiPage, a serialized Flow, a minified bundle and a one-line JSON document all put the whole file on line 1, so every ref on that line stored the entire file as its context. Measured on a 779 KB FlexiPage with 10,000 component references: 7,789,990,000 characters written for a 779 KB input, quadratic in the file and the reason indexing it took 58 s. Keeping the head of the line would be no better than eliding it, since the reference itself is usually nowhere near character 0; the window is centred on `col` so the row still shows what it is a reference to. Bounded here, at the single write path, for the same reason {@link boundSymbolBody} is: it bounds every current and future extractor rather than the one that surfaced it. */
export function boundRefContext(context: string, col: number): string {
if (context.length <= MAX_REF_CONTEXT_CHARS) return context
const budget = MAX_REF_CONTEXT_CHARS - REF_CONTEXT_ELISION.length * 2
// `col` is measured against the untrimmed source line while `context` is trimmed, so it is an approximate offset into this string; clamping is what makes an approximate centre safe rather than a source of empty windows.
const centre = Math.min(Math.max(col, 0), context.length)
let start = Math.max(0, Math.min(centre - Math.floor(budget / 2), context.length - budget))
let end = start + budget
// Never cut between the halves of a surrogate pair at either edge: storing a lone surrogate is not valid text and surfaces downstream as a replacement character.
if (start > 0 && context.charCodeAt(start) >= 0xdc00 && context.charCodeAt(start) <= 0xdfff) start += 1
if (end < context.length && context.charCodeAt(end - 1) >= 0xd800 && context.charCodeAt(end - 1) <= 0xdbff) end -= 1
return (start > 0 ? REF_CONTEXT_ELISION : '') + context.slice(start, end) + (end < context.length ? REF_CONTEXT_ELISION : '')
}
// precedingDocComment / DocCommentStyle live in doc_comment.ts (shared with languages/common.ts, which parser.ts itself imports from -- defining them here would create an import cycle). See that module's doc comment. Re-exported here so existing importers of `parser.js` keep working.
export { precedingDocComment, type DocCommentStyle } from './doc_comment.js'
function loadGrammar(lang: Language, filePath?: string, content?: string): Grammar | null {
const useTsx = lang === 'typescript' && filePath !== undefined && path.extname(filePath).toLowerCase() === '.tsx'
const useCppHeader =
lang === 'c' &&
filePath !== undefined &&
path.extname(filePath).toLowerCase() === '.h' &&
content !== undefined &&
CPP_HEADER_SNIFF_RE.test(content)
const cacheKey = useTsx ? 'typescript:tsx' : useCppHeader ? 'c:cpp-header' : lang
const cached = _grammarCache.get(cacheKey)
if (cached !== undefined) return cached
let grammar: Grammar | null = null
try {
if (lang === 'typescript') {
const mod = _require('tree-sitter-typescript') as { typescript: Grammar; tsx: Grammar }
grammar = useTsx ? mod.tsx : mod.typescript
} else if (lang === 'javascript') {
grammar = _require('tree-sitter-javascript') as Grammar
} else if (lang === 'python') {
grammar = _require('tree-sitter-python') as Grammar
} else if (lang === 'go') {
grammar = _require('tree-sitter-go') as Grammar
} else if (lang === 'rust') {
grammar = _require('tree-sitter-rust') as Grammar
} else if (lang === 'ruby') {
grammar = _require('tree-sitter-ruby') as Grammar
} else if (lang === 'java') {
grammar = _require('tree-sitter-java') as Grammar
} else if (lang === 'c') {
grammar = useCppHeader ? (_require('tree-sitter-cpp') as Grammar) : (_require('tree-sitter-c') as Grammar)
} else if (lang === 'cpp') {
grammar = _require('tree-sitter-cpp') as Grammar
}
} catch {
grammar = null
}
_grammarCache.set(cacheKey, grammar)
return grammar
}
/** Is tree-sitter (binding + grammar) available for `lang`? Returns `false` rather than throwing when the native binding or a grammar package is missing, so callers can branch to the regex fallback. Languages without a bundled grammar (markdown, json, yaml, toml, css, dockerfile, bash, unknown) are always `false`. */
export function isTreeSitterAvailable(lang: Language): boolean {
if (!TREE_SITTER_LANGUAGES.includes(lang)) return false
return loadParserCtor() !== null && loadGrammar(lang) !== null
}
/** The grammar packages `loadGrammar` requires. */
const TREE_SITTER_GRAMMAR_PACKAGES: readonly string[] = [
'tree-sitter-typescript',
'tree-sitter-javascript',
'tree-sitter-python',
'tree-sitter-go',
'tree-sitter-rust',
'tree-sitter-ruby',
'tree-sitter-java',
'tree-sitter-c',
'tree-sitter-cpp',
]
/** Grammar packages that do not resolve from the bundle's location; resolution only, so it answers even when the core binding cannot load. */
export function missingTreeSitterGrammarPackages(): string[] {
return TREE_SITTER_GRAMMAR_PACKAGES.filter((pkg) => {
try {
_require.resolve(pkg)
return false
} catch {
return true
}
})
}
// --- Public parse / index API -----------------------------------------------
/** Parse one file and return its extracted symbols + refs. Dispatches to the tree-sitter extractor when a grammar is available for the detected language, otherwise falls back to regex. Unknown languages and unreadable files yield empty symbol/ref lists (never throws). Call-site refs are extracted for the tree-sitter languages in {@link REF_LANGUAGES}; the regex-fallback and structured-config languages yield no refs. */
export async function parseFile(filePath: string): Promise<ParseResult> {
await loadRegexExtractors()
const start = Date.now()
const pathLanguage = detectLanguage(filePath)
let content: string
try {
content = decodeSource(await fs.promises.readFile(filePath))
} catch {
return { symbols: [], refs: [], language: pathLanguage, duration: Date.now() - start }
}
const language = refineLanguageByContent(filePath, pathLanguage, content)
const { symbols, refs } = parseContent(content, filePath, language)
return { symbols, refs, language, duration: Date.now() - start }
}
/** Symbols + refs extracted from one file's content. */
interface ParseContentResult {
readonly symbols: SymbolEntry[]
readonly refs: RefEntry[]
}
function isLwcFile(filePath: string, extension: '.js' | '.html'): boolean {
const normalized = filePath.replace(/\\/g, '/')
return /\/lwc\/[^/]+\/[^/]+$/i.test(normalized) && normalized.toLowerCase().endsWith(extension)
}
function mergeParseResults(...results: readonly ParseContentResult[]): ParseContentResult {
const symbols: SymbolEntry[] = []
const refs: RefEntry[] = []
const seenSymbols = new Set<string>()
const seenRefs = new Set<string>()
for (const result of results) {
for (const entry of result.symbols) {
const key = `${entry.filePath}\0${entry.name}\0${entry.kind}\0${entry.lineStart}`
if (seenSymbols.has(key)) continue
seenSymbols.add(key)
symbols.push(entry)
}
for (const entry of result.refs) {
const key = `${entry.filePath}\0${entry.name}\0${entry.line}\0${entry.col}`
if (seenRefs.has(key)) continue
seenRefs.add(key)
refs.push(entry)
}
}
return { symbols, refs }
}
/** Shared sync core: pick an extractor for `language` and run it on `content`. */
function parseContent(content: string, filePath: string, language: Language): ParseContentResult {
// Strip a UTF-8 BOM (U+FEFF) and read a lone CR as a line break: both entry points (parseFile, indexFileSync) funnel through here, so this is the single place it needs to happen. A CR-only file otherwise put every symbol on line 1. Sha/hash computation elsewhere stays on the raw original bytes — only this decoded copy is affected.
content = normalizeSourceText(content)
// A notebook is JSON, not source -- flatten its code/markdown cells into a virtual Python-like document and recurse with language forced to 'python' so it gets the real tree-sitter Python extraction path; a non-Python-kernel or unparseable notebook yields no symbols/refs (never throws).
if (language === 'ipynb') {
const virtual = ipynbToVirtualSource(content)
if (virtual.cellLanguage === null) return { symbols: [], refs: [] }
return parseContent(virtual.content, filePath, 'python')
}
const viaTreeSitter = parseWithTreeSitter(content, filePath, language)
if (viaTreeSitter !== null) return viaTreeSitter
// Regex-based extractors for languages without tree-sitter
return extractNoTreeSitter(content, filePath, language)
}
/** The tree-sitter half of {@link parseContent}, split out so a caller that must never be handed the regex fallback can ask for tree-sitter alone. `null` means tree-sitter did not produce this file's symbols: no grammar or no parser constructor for the language, or the parse threw. It never means "this file has no symbols", which is the empty array. {@link parseSourceSymbolsTreeSitterOnly} is the caller that needs that distinction, while {@link parseContent} itself reads null as "fall through to the regex pass", exactly as it did when this block was inline. */
function parseWithTreeSitter(content: string, filePath: string, language: Language): ParseContentResult | null {
if (!TREE_SITTER_LANGUAGES.includes(language)) return null
let grammar: Grammar | null = null
let parser: TsParser | undefined
try {
const Ctor = loadParserCtor()
if (Ctor === null) return null
grammar = loadGrammar(language, filePath, content)
if (grammar === null) return null
parser = _parserByGrammar.get(grammar)
if (parser === undefined) {
parser = new Ctor()
parser.setLanguage(grammar)
_parserByGrammar.set(grammar, parser)
}
const tree = parser.parse(content)
const root = tree.rootNode
let symbols: SymbolEntry[]
if (language === 'python') {
symbols = extractPythonSymbols(root, filePath)
} else if (language === 'go') {
symbols = extractGoSymbols(root, filePath, content.split(/\r?\n/))
} else if (language === 'rust') {
symbols = extractRustSymbols(root, filePath, content.split(/\r?\n/))
} else if (language === 'ruby') {
symbols = extractRubySymbols(root, filePath, content.split(/\r?\n/))
} else if (language === 'java') {
symbols = extractJavaSymbols(root, filePath, content.split(/\r?\n/))
} else if (language === 'cpp' || language === 'c') {
symbols = extractCppSymbols(root, filePath, content.split(/\r?\n/))
} else {
symbols = extractTsJsSymbols(root, filePath, content.split(/\r?\n/))
}
const refs = REF_LANGUAGES.has(language) ? extractRefs(root, filePath, language) : []
const parsed = { symbols, refs }
return language === 'javascript' && isLwcFile(filePath, '.js')
? mergeParseResults(parsed, salesforceFrontend().extractLwcJavaScript(content, filePath))
: parsed
} catch {
// Parser threw on this input: the caller decides whether a regex pass is an acceptable substitute for what it could not produce.
if (grammar !== null && parser !== undefined) _parserByGrammar.delete(grammar)
return null
}
}
/** Symbols for one file's content from tree-sitter and nothing else, or `null` when tree-sitter could not supply them. This exists for the post-read source-skeleton fold in hooks_read.ts, which composes what the model reads out of the symbols returned here. A regex fallback would be wrong there in a way it is not wrong for the index: `extractWithRegex` finds 40-57% of what tree-sitter finds on the same files (measured: 9 against 21, 33 against 82, 48 against 85), and a skeleton built from a partial symbol list omits declarations with nothing in the output to signal the omission. An incomplete map is worse than no map, so that caller delivers the file whole on `null`. Takes content rather than a path on purpose: the fold runs on the FIRST read of a file the indexer may never have touched, and the bytes it has to describe are the ones the harness just delivered, not whatever is on disk now. */
export function parseSourceSymbolsTreeSitterOnly(content: string, filePath: string, language: Language): SymbolEntry[] | null {
// The same normalization parseContent does, for the same reason: a BOM would shift every tree-sitter offset by one, and a lone CR would put every symbol on line 1.
const parsed = parseWithTreeSitter(normalizeSourceText(content), filePath, language)
return parsed === null ? null : parsed.symbols
}
type SymbolExtractor = (content: string, filePath: string) => SymbolEntry[]
// In the global symbol registry so a module reset (vi.resetModules) or a second copy of this module in one process still finds the adapters loaded.
const REGEX_ADAPTERS_SLOT = Symbol.for('token-goat.regex-adapters')
const adapterSlot = globalThis as unknown as Record<symbol, typeof RegexAdapters | undefined>
const SALESFORCE_FRONTEND_SLOT = Symbol.for('token-goat.salesforce-frontend')
const salesforceSlot = globalThis as unknown as Record<symbol, typeof SalesforceFrontend | undefined>
/** Load the regex language adapters (src/languages/registry.ts), once per process. They sit behind a dynamic import because parser.ts is on the hook path, where none of them is ever called: imported statically, every adapter was compiled on every hook invocation. Each entry point that parses awaits this first (the CLI's run(), runWorkerLoop, parseFile); a sync parse without it throws rather than quietly indexing a file to nothing. */
export async function loadRegexExtractors(): Promise<void> {
adapterSlot[REGEX_ADAPTERS_SLOT] ??= await import('./languages/registry.js')
salesforceSlot[SALESFORCE_FRONTEND_SLOT] ??= await import('./languages/salesforce_frontend.js')
}
// The Salesforce LWC/Aura extractors load with the regex adapters, so the hook path never compiles them.
function salesforceFrontend(): typeof SalesforceFrontend {
const loaded = salesforceSlot[SALESFORCE_FRONTEND_SLOT]
if (loaded === undefined) throw new Error('token-goat: the Salesforce frontend adapters are not loaded; await loadRegexExtractors() before parsing')
return loaded
}
function regexAdapters(): typeof RegexAdapters {
const loaded = adapterSlot[REGEX_ADAPTERS_SLOT]
if (loaded === undefined) throw new Error('token-goat: the regex language adapters are not loaded; await loadRegexExtractors() before parsing')
return loaded
}
let noTreeSitterTable: Record<RegexLanguage, SymbolExtractor> | undefined
// One entry per `regex` row of src/language_specs.ts, required by the type: the adapter rows come from registry.ts's ADAPTER_EXTRACTORS, whose own type fails on a row without an extractor.
function noTreeSitterExtractors(): Record<RegexLanguage, SymbolExtractor> {
noTreeSitterTable ??= {
markdown: extractMarkdownSymbols,
json: extractJsonSymbols,
yaml: extractYamlSymbols,
toml: extractTomlSymbols,
css: extractCssSymbols,
dockerfile: extractDockerfileSymbols,
...regexAdapters().ADAPTER_EXTRACTORS,
}
return noTreeSitterTable
}
function extractNoTreeSitter(
content: string,
filePath: string,
language: Language,
): ParseContentResult {
const adapters = regexAdapters()
if (language === 'salesforce_metadata') return adapters.extractSalesforceMetadata(content, filePath)
if (language === 'salesforce_markup') return salesforceFrontend().extractSalesforceMarkup(content, filePath)
if (language === 'html' && isLwcFile(filePath, '.html')) {
const base: ParseContentResult = { symbols: adapters.ADAPTER_EXTRACTORS.html(content, filePath), refs: [] }
return mergeParseResults(base, salesforceFrontend().extractLwcTemplate(content, filePath))
}
// Vue/Svelte/Astro adapters emit both symbols and refs (template component-tag references), same shape as extractSalesforceMarkup above -- returned directly rather than forced through the symbols-only noTreeSitterExtractors table.
if (language === 'vue') return adapters.extractVue(content, filePath)
if (language === 'svelte') return adapters.extractSvelte(content, filePath)
if (language === 'astro') return adapters.extractAstro(content, filePath)
// COBOL and Natural also emit refs (PERFORM, GO TO, CALL/CALLNAT/FETCH literals); they stay outside REF_LANGUAGES because paragraphs are reached by fall-through and programs by name, so an empty ref set is no evidence of dead code.
if (language === 'cobol') {
const r = adapters.extractCobol(content, filePath)
return { symbols: r.symbols, refs: r.refs }
}
if (language === 'natural') {
const r = adapters.extractNatural(content, filePath)
return { symbols: r.symbols, refs: r.refs }
}
if (language === 'csharp') {
const r = adapters.extractCsharp(content, filePath)
const symbols = adapters.spanUnsettled(r.symbols, r.settled, (rest) => adapters.assignBraceBlockSpans(rest, content, adapters.CSHARP_SPAN_OPTS))
return { symbols, refs: r.refs }
}
const parsed: ParseContentResult = {
symbols: extractSymbolsNoTreeSitter(content, filePath, language),
refs: [],
}
return language === 'javascript' && isLwcFile(filePath, '.js')
? mergeParseResults(parsed, salesforceFrontend().extractLwcJavaScript(content, filePath))
: parsed
}
/** Symbol extraction for languages with no tree-sitter grammar: the regex and structured-config adapters. Returns an empty list for `unknown`. */
function extractSymbolsNoTreeSitter(
content: string,
filePath: string,
language: Language,
): SymbolEntry[] {
if (language === 'unknown') return []
// A tree-sitter language whose grammar did not load has no entry and falls back to the coarse regex scan.
return ((noTreeSitterExtractors() as Partial<Record<Language, SymbolExtractor>>)[language] ?? extractWithRegex)(content, filePath)
}
/** Delete every index row (symbols, refs, files) for one file. On a case-insensitive filesystem the path match folds case — mirroring index_reader's pathEq — so rows written under a different path casing by a prior reindex are removed rather than orphaned as case-variant duplicates. */
export function deleteFileRows(db: ReturnType<typeof getDb>, filePath: string): void {
const folded = foldPath(filePath)
db.prepare(`DELETE FROM symbols WHERE ${pathEqClause('file_path')}`).run(folded)
db.prepare(`DELETE FROM refs WHERE ${pathEqClause('file_path')}`).run(folded)
db.prepare(`DELETE FROM files WHERE ${pathEqClause('path')}`).run(folded)
}
/** True when the basename is a minified bundle by name: `.min.js`, `.min.mjs`, `.min.cjs`, `.min.css` or `-min.js`, case-insensitive. Name-based on purpose: it is deterministic and cannot misfire on hand-written code. */
export function isMinifiedBundlePath(filePath: string): boolean {
return /(?:\.min\.(?:js|mjs|cjs|css)|-min\.js)$/i.test(path.basename(filePath))
}
/** True when any directory segment of `filePath` BELOW the project root matches a basename in `skipDirs` -- the `indexing.skip_dirs` config knob. The segments above the root are where the user keeps the project, not part of it: a project at `C:/work/build/app` is not vendored output. `projectRoot` is the root the caller knows (cmdIndex's walk root); omitted, it is derived from project markers (see skip_scope.ts). Splits on either separator since callers pass both forward-slash-normalized keys (resolveIndexPath) and raw absolute paths. */
export function isUnderSkipDir(filePath: string, skipDirs: readonly string[], projectRoot?: string): boolean {
if (skipDirs.length === 0) return false
return hasSkipSegmentBelowRoot(filePath, projectRoot, (seg) => skipDirs.includes(seg))
}
/** True when `filePath` is excluded from the syntactic parse entirely by `indexing.skip_dirs`, a generated-report basename configured via `indexing.skip_files` (defaults to coverage.json / coverage-final.json -- single-line minified JSON blobs whose indexed "property" symbols bloat the index and trip the oversized-symbol doctor check for zero benefit), or `indexing.large_file_skip_kb`. Must be evaluated UNCONDITIONALLY (independent of any sha/parseUnchanged gate) because a file that becomes skip-eligible via a config change alone must still have its stale rows purged. A stat failure is treated as "not skip-eligible". */
export function isParseSkipEligible(filePath: string, cfg: IndexingConfig, projectRoot?: string): boolean {
if (isUnderSkipDir(filePath, cfg.skip_dirs, projectRoot)) return true
if (cfg.skip_files.includes(path.basename(filePath))) return true
if (cfg.skip_minified !== false && isMinifiedBundlePath(filePath)) return true
try {
const stat = fs.statSync(filePath)
if (stat.size > cfg.large_file_skip_kb * 1024) return true
} catch {
// let the caller's own read/stat attempt handle/report the failure
}
return false
}
/** Canonical form of one embedding-boundary set: a sorted multiset of the symbol spans buildEmbeddingBoundaries cuts a non-markdown file on, so the comparison is independent of row order and of rowid. */
function boundarySpanKey(spans: readonly { start: number; end: number }[]): string {
return spans.map((s) => `${s.start}-${s.end}`).sort().join(',')
}
/** True when the symbol spans `result` is about to write differ from the ones already stored for `filePath` -- i.e. this reparse moves the cuts the file's existing chunks were embedded on. Must be called BEFORE deleteFileRows removes the prior rows. Mirrors writeParseResult's own insert filter so a symbol it declines to store is not counted as a boundary on either side. */
function embeddingBoundariesMoved(db: SqliteDatabase, filePath: string, result: ParseResult): boolean {
const priorSpans = db
.prepare(`SELECT line_start, line_end FROM symbols WHERE ${pathEqClause('file_path')}`)
.all(foldPath(filePath)) as { line_start: number; line_end: number }[]
const prior = boundarySpanKey(priorSpans.map((r) => ({ start: r.line_start, end: r.line_end })))
const next = boundarySpanKey(
result.symbols
.filter((s) => s.name !== '' && s.kind !== '')
.map((s) => ({ start: s.lineStart, end: s.lineEnd })),
)
return prior !== next
}
/** Write a parsed result's rows into the index DB, replacing any prior rows for the file in a single transaction (DELETE + INSERT, matching the Python bulk-replace strategy) so a re-index never leaves stale symbols behind. Called by {@link indexFileSync}, the worker drain loop's synchronous entry point. */
function writeParseResult(
filePath: string,
content: Buffer,
result: ParseResult,
dbPath: string,
precomputedSha?: string,
): void {
const db = getDb(dbPath)
// Hash the SAME raw bytes that were actually parsed, not a fresh disk re-read: if the file changes between the parse read and this write, a re-read here would record a SHA that does not match the symbols/refs actually written below, and the worker's SHA-gated incremental drain would skip reindexing a file whose stored SHA happens to match a later version, leaving it permanently stuck with stale symbols. Takes the raw Buffer (not the utf8-decoded string used for parsing) so this SHA is computed over the same bytes worker.ts's gate hashes via fingerprintFile() -- a lossy utf8 decode/re-encode round-trip on invalid-UTF-8 content would otherwise produce a different digest than hashing the raw bytes directly, permanently defeating the gate for any such file. writeParseResult's only caller (indexFileSync) always has a successfully-read Buffer in hand by the time it calls this -- a read failure returns or throws before reaching this call -- so there is never a "content is unreadable" case to fall back to a disk re-read for.
const sha = precomputedSha ?? fingerprintContent(content)
const mtime = safeMtime(filePath)
const now = Date.now() / 1000
// embed_sha is the OTHER freshness key this same row carries (see files.embed_sha / makeIndexer in worker.ts): it answers "were the current chunks/vectors embedded from this exact content", independent of files.parser_sha answering "did this extractor version write the symbol/ref rows below". A parser-only reparse -- this write happening because parser_sha was stale while the bytes on disk never moved -- must not silently reset that independent answer to unknown. Read the prior row BEFORE deleteFileRows below removes it: once the DELETE runs there is nothing left to read this from, and the whole point is to carry it across that delete.
const priorRow = db
.prepare(`SELECT sha, embed_sha FROM files WHERE ${pathEqClause('path')}`)
.get(foldPath(filePath)) as { sha: string | null; embed_sha: string | null } | undefined
// Preserve only when the CONTENT this row describes is unchanged (sha match): a content change means the old embed_sha was computed from bytes that no longer exist, and carrying it forward would make makeIndexer's `isEmbedFresh` check believe stale vectors are still valid for the new content. When sha matches, the chunks embeddings.ts wrote for it are untouched by this reparse (deleteFileRows never touches chunks/chunk_vectors), so the stamp describing them is still true and re-embedding identical content for a parser-only bump would be pure waste.
const carriedEmbedSha = priorRow !== undefined && priorRow.sha === sha ? priorRow.embed_sha : null
// ...unless this reparse MOVED the boundaries those chunks were cut on. buildEmbeddingBoundaries draws a non-markdown file's chunk cuts straight from its symbol rows, and this write is about to replace them, so a reparse that extracts a different symbol set leaves the stored chunks (and their vectors) drawn on cuts that no longer exist. That is what a language adapter edit produces: it moves the language's PARSER_FINGERPRINT and reparses the file, while EMBED_FINGERPRINT deliberately stays put -- the global digest cannot be the answer here, because moving it re-embeds every already-embedded file on the machine for what is a per-language, often per-file change. Comparing the file's own boundary list across the reparse is that per-file answer, and it is exact in both directions: identical boundaries still carry the stamp, moved boundaries drop it so the file is re-embedded exactly once. Markdown is excluded because its boundaries come from headings in the content, which the sha match above has already established is unchanged. Only a bare-sha stamp is examined: every marker stamp (disabled:/unavailable:/oversize:/asset:/...) describes a file that has no boundary-cut chunks at all, so a boundary move says nothing about it, and dropping it would only re-enter indexFileEmbeddings to re-derive the same marker.
const embedShaToCarry =
carriedEmbedSha !== null &&
carriedEmbedSha === sha &&
detectLanguage(filePath) !== 'markdown' &&
embeddingBoundariesMoved(db, filePath, result)
? null
: carriedEmbedSha
const writeAll = db.transaction(() => {
deleteFileRows(db, filePath)
// parser_sha records WHICH extraction logic produced the symbol and ref rows written just below, so a later parser change can tell that these rows are stale even though the content sha still matches. Without it, files.sha was the only freshness key and answered only "has the content changed", which left an unedited file pinned to the symbol set an older parser gave it for as long as nobody touched it. Stamped here rather than in the gates so it is written by exactly the transaction that writes the rows it describes -- and derived from the same result.language that goes into the row beside it, so the gates, which look the expected stamp up by that stored column, can never be asking about a different language than the one that actually parsed the file.
db.prepare(
'INSERT INTO files (path, sha, mtime, language, indexed_at, parser_sha, embed_sha) VALUES (?, ?, ?, ?, ?, ?, ?)',
).run(filePath, sha, mtime, result.language, now, parserFingerprintForLanguage(result.language), embedShaToCarry)
const insSym = db.prepare(
'INSERT INTO symbols (file_path, name, kind, line_start, line_end, body, docstring, parent) ' +
'VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
)
for (const s of result.symbols) {
if (s.name === '' || s.kind === '') continue
// Bound every stored body regardless of which extractor produced it -- see MAX_SYMBOL_BODY_CHARS / boundSymbolBody. This is the single choke point through which all parsed symbols reach the DB, so capping here makes an unbounded-body bug in any one language extractor incapable of bloating global.db. An over-cap body is stored empty, not truncated, so resolveBody still serves the complete symbol from source on read.
insSym.run(
s.filePath,
s.name,
s.kind,
s.lineStart,
s.lineEnd,
boundSymbolBody(s.body),
boundSymbolDocstring(s.docstring),
s.parent,
)
}
const insRef = db.prepare(
'INSERT INTO refs (file_path, name, line, col, context) VALUES (?, ?, ?, ?, ?)',
)
for (const r of result.refs) {
if (r.name === '' || COMPILER_ARTIFACT_REF_NAMES.has(r.name)) continue
insRef.run(r.filePath, r.name, r.line, r.col, boundRefContext(r.context, r.col))
}
})
// `.immediate()` -- BEGIN IMMEDIATE -- rather than a plain call, which the driver issues as a deferred BEGIN. A deferred transaction takes its read snapshot first and only asks for the write lock when it reaches a writing statement, and SQLite refuses that upgrade with SQLITE_BUSY straight away instead of consulting the busy handler, so `busy_timeout` does nothing for it. Six `index` runs starting together against a database that did not exist yet reproduced it every time: one of them failed on its very first file with "database is locked", dropped that file from the index, and still exited 0. Raising busy_timeout to 60s in a run lasting one second changed nothing, which is what proved the handler was never being asked. BEGIN IMMEDIATE takes the write lock up front, where the busy handler does apply.
writeAll.immediate()
}
/** Synchronous index: consume a supplied byte snapshot or read, parse, and persist one file's rows. An ENOENT on the direct-read path is skipped; other read failures propagate to the caller. */
// Confinement-pinned read_commands callers pass bytes only after checking file identity; the worker passes the same bytes and their precomputed SHA from its freshness gate so parsing and persistence neither reopen nor rehash that snapshot. Other callers omit the snapshot and retain this function's own read and hash.
export function indexFileSync(
rawPath: string,
dbPath: string = globalDbPath(),
preReadBytes?: Buffer,
preReadSha?: string,
): void {
// Every symbols/refs/files row this call writes derives from this one spelling, so the whole parse side of the index agrees on a single name for the file. See canonicalizeIndexPath.
const filePath = canonicalizeIndexPath(rawPath)
const ixCfg = loadConfig().indexing
if (ixCfg !== undefined && isParseSkipEligible(filePath, ixCfg)) {
// Purge stale rows AND the files row (sha) so the file settles into a stable not-indexed state instead of being re-selected as "changed" on every drain; also drop any embedding rows it held before becoming skip-eligible (indexFileSync is called directly from read_commands' --force-refresh path).
const db = getDb(dbPath)
deleteFileRows(db, filePath)
deleteFileEmbeddings(db, filePath)
return
}
const pathLanguage = detectLanguage(filePath)
let raw: Buffer
if (preReadBytes !== undefined) {
raw = preReadBytes
} else {
try {
raw = fs.readFileSync(filePath)
} catch (err) {
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return
throw err
}
}
const content = decodeSource(raw)
const language = refineLanguageByContent(filePath, pathLanguage, content)
const { symbols, refs } = parseContent(content, filePath, language)
writeParseResult(
filePath,
raw,
{ symbols, refs, language, duration: 0 },
dbPath,
preReadBytes === undefined ? undefined : preReadSha,
)
}
// buildEmbeddingBoundaries moved to embedding_boundaries.ts so EMBED_FINGERPRINT can hash exactly the sources that decide chunk boundaries without pulling in the rest of this file, and without PARSER_FINGERPRINT hashing embedding-only code. Re-exported here so every existing caller and test importing it from parser.ts keeps working.
export { buildEmbeddingBoundaries }
/** Prefix used to stamp `files.embed_sha` when {@link indexFileEmbeddings} early-returns because `indexing.embeddings_enabled` is off, instead of the file's real content sha. Kept distinct from a real embed_sha (see {@link indexFileEmbeddings}'s own doc comment) so re-enabling embeddings later can't be mistaken for "already embedded, unchanged" by makeIndexer's embedUnchanged gate in worker.ts, which checks the CURRENT embeddings_enabled state and only treats this disabled-marker form as "unchanged" while still disabled -- a bare sha match here would otherwise permanently skip re-embedding once a user turns embeddings back on for content that was only ever marker-stamped, never actually embedded. */
export const DISABLED_EMBED_SHA_PREFIX = 'disabled:'
/** The embed_sha value {@link indexFileEmbeddings} stamps for `sha` while embeddings are disabled. */
export function disabledEmbedSha(sha: string): string {
return DISABLED_EMBED_SHA_PREFIX + sha
}
let disabledStamps = 0
/** How many `disabled:` markers {@link indexFileEmbeddings} has stamped in this process. The worker compares it across idle cycles: a marker stamped since the last one means embeddings were off at some moment in between, even when every cycle that looked found them on. */
export function disabledStampCount(): number {
return disabledStamps
}
/** Prefix used to stamp `files.embed_sha` when {@link indexFileEmbeddings} could not actually embed a file because the optional embedding deps were absent (the inference runtime or the sqlite-vec `chunk_vectors` table). Distinct from a real embed_sha so the freshness gate can re-embed the file once the deps are installed -- otherwise a project indexed on a deps-less install would stamp bare shas, and every previously-indexed unchanged file would look "already embedded" the instant the model is added, leaving the semantic index permanently empty for that content. Distinct from {@link DISABLED_EMBED_SHA_PREFIX} because the two cases clear on different conditions: disabled clears when config re-enables embeddings, unavailable clears when the deps become installable/usable (see {@link isEmbedFresh}). */
export const UNAVAILABLE_EMBED_SHA_PREFIX = 'unavailable:'
/** The embed_sha value {@link indexFileEmbeddings} stamps for `sha` when embedding deps are absent. */
export function unavailableEmbedSha(sha: string): string {
return UNAVAILABLE_EMBED_SHA_PREFIX + sha
}
/** Prefix used to stamp `files.embed_sha` when {@link indexFileEmbeddings} skipped a file only because it is larger than `indexing.large_file_symbol_only_kb`. Third instance of the same shape as {@link DISABLED_EMBED_SHA_PREFIX} and {@link UNAVAILABLE_EMBED_SHA_PREFIX}, and it clears on a third condition: the threshold itself, which is a user-tunable config value rather than a property of the content. A bare sha here reads as "really embedded, nothing left to do", so raising the threshold left every previously-skipped file looking permanently fresh and the background worker never re-embedded any of them -- `semantic` stayed blind to that content until something edited it, while `symbol` and `read` worked normally. `token-goat doctor`'s embedding coverage remediation tells users to raise exactly this value, so the case is reached by following the product's own advice. The threshold in force at stamp time is encoded in the marker so a later change to it no longer matches and the file is re-examined; see {@link isEmbedFresh}. */
export const OVERSIZE_EMBED_SHA_PREFIX = 'oversize:'
/** The embed_sha value {@link indexFileEmbeddings} stamps for `sha` when the file is over `symbolOnlyKb`. */
export function oversizeEmbedSha(sha: string, symbolOnlyKb: number): string {
return `${OVERSIZE_EMBED_SHA_PREFIX}${symbolOnlyKb}:${sha}`
}
/** Prefix used to stamp `files.embed_sha` when {@link indexFileEmbeddings} gave up on a document because its extraction ran past {@link MAX_DOCUMENT_WORK_MILLIS}. Fourth instance of the shape {@link OVERSIZE_EMBED_SHA_PREFIX} introduced, and it exists for the same reason: the skip was conditional on a value that can change, so a bare sha would record it as terminal and the document would never be embedded again. A clock refusal is the weakest of all these conditions, because the value it is conditional on is not even a setting -- it is how busy this machine was for one minute. Stamping a bare sha for it, which is what this branch used to do, is exactly the "permanent verdict recorded from a temporary condition" the extraction-*failure* branch a few lines below refuses to do, applied to a case that only looked settled because the error happened to extend {@link DocumentRefusedError}. The bound in force is encoded so that raising the clock in a later release re-examines every document a lower one refused; see {@link isEmbedFresh}. */
export const TIMEOUT_EMBED_SHA_PREFIX = 'timeout:'
/** The embed_sha value {@link indexFileEmbeddings} stamps for `sha` when extraction passed `workMillis`. */
export function timeoutEmbedSha(sha: string, workMillis: number): string {
return `${TIMEOUT_EMBED_SHA_PREFIX}${workMillis}:${sha}`
}
/** Prefix used to stamp `files.embed_sha` when {@link indexFileEmbeddings} skipped a file because its extension names a format with no text in it (see NON_TEXT_ASSET_EXTENSIONS in asset_extensions.ts). Bears {@link NON_TEXT_ASSET_SET_ID} for the reason {@link TIMEOUT_EMBED_SHA_PREFIX} bears its work clock: the bound is a compiled-in list rather than a config value, and a release that edits the list must re-examine every file stamped under the old one -- otherwise removing an extension from the set would leave every file that already carried that extension buried forever, the same permanent-verdict-from-a-changed-condition failure a bare sha causes for the oversize case. */
export const ASSET_EMBED_SHA_PREFIX = 'asset:'
/** The embed_sha value {@link indexFileEmbeddings} stamps for `sha` when the file is a non-text asset. */
export function assetEmbedSha(sha: string): string {
return `${ASSET_EMBED_SHA_PREFIX}${NON_TEXT_ASSET_SET_ID}:${sha}`
}
/** Prefix used to stamp `files.embed_sha` when {@link indexFileEmbeddings} skipped a file that would have contributed more than `indexing.max_chunks_per_file` chunks. Threshold-bearing for the same reason {@link OVERSIZE_EMBED_SHA_PREFIX} is: the bound is user-tunable, so raising it has to re-examine every file the old value refused rather than leave them stamped as done. Distinct from the oversize marker rather than reusing it because the two measure different axes and a file can be well under the byte threshold while far over this one. Chunk cuts snap to structure, so a generated JSON snapshot of a few hundred kilobytes -- thousands of one-line keys, each its own boundary -- becomes thousands of near-identical chunks, while a source file of the same size becomes a few hundred meaningful ones. On one real index the twenty-odd worst files were all under the 500 KB byte threshold and held 22% of every chunk on the machine. */
export const MAX_CHUNKS_EMBED_SHA_PREFIX = 'maxchunks:'
/** The embed_sha value {@link indexFileEmbeddings} stamps for `sha` when the file exceeds `maxChunks`. */
export function maxChunksEmbedSha(sha: string, maxChunks: number): string {
return `${MAX_CHUNKS_EMBED_SHA_PREFIX}${maxChunks}:${sha}`
}
/** Apply `maxChunks` to the rows a just-finished embed actually wrote, dropping them and stamping the over-cap marker when it went over. Returns whether it did, so the caller can skip its own stamp. The cheap count taken before embedding cannot decide this on its own. It cuts with `chunkFile`'s char budget, and `indexFile` re-cuts with the model's tokenizer before inserting whenever the embedding deps are present -- the branch that exists precisely when this cap matters. The token cut is strictly the finer of the two: it flushes on a token budget as well as a char one, it sub-splits any single line past that budget, and it stops merging below-floor chunks ("a short vector costs one more row", as the flush in embeddings.ts puts it). So the pre-count is a lower bound, and a file can clear a 600-chunk ceiling on the coarse cut and store several hundred more than that on the fine one. The pre-count is kept because it is free and its rejections are all correct; it is just not the last word. Counting stored rows is also what makes this agree with `selectUnembeddableChunkFiles` (embed_backfill.ts), which sweeps on exactly this measure. When the two used different measures they could reach opposite verdicts on one file, and which one won came down to whether that sweep's version ledger had been stamped yet. */
function enforceStoredChunkCap(db: SqliteDatabase, filePath: string, sha: string | undefined, maxChunks: number): boolean {
// No sha means stampEmbedSha is a no-op, so there would be no marker to record the verdict. Dropping the rows anyway would leave the file looking un-embedded and send the next pass straight back through the same inference, so leave it alone and let the embed_backfill sweep take it.
if (sha === undefined || maxChunks <= 0) return false
const row = db.prepare(`SELECT COUNT(*) AS n FROM chunks WHERE ${pathEqClause('file_path')}`).get(foldPath(filePath)) as
| { n: number }
| undefined
if ((row?.n ?? 0) <= maxChunks) return false
deleteFileEmbeddings(db, filePath)
stampEmbedSha(db, filePath, sha, (s) => maxChunksEmbedSha(s, maxChunks))
return true
}
/** Return `absPath` with its final segment spelled the way the filesystem actually spells it. The companion of `indexedPathSpellingIsStale`. That guard notices when a stored row's spelling has drifted from disk and forces a reparse -- but a reparse only helps if the reparse writes a *different* spelling than the one already stored. Every writer used to hand its own caller's path straight to `writeParseResult`, and no caller's spelling is authoritative: `git ls-files` reports the name in git's index, which still says `MixedName.ts` after an unstaged case-only rename on a case-insensitive filesystem, because git never sees such a rename at all. So the bulk `index` command detected the drift, reparsed, wrote the same stale name back, and detected the very same drift again on the next run -- reparsing and re-embedding an unchanged file on every single run, forever, while every citation named a path with no directory entry behind it. Canonicalizing here, at the one point every writer passes through, is what makes that converge. Only the last segment is taken from the filesystem. `fs.realpathSync.native` canonicalizes every parent directory too, so adopting its whole answer would rewrite rows to an ambient prefix spelling the caller never used (`C:/WINDOWS/TEMP/...` for a caller that said `C:/Windows/Temp`), churning paths for a difference that is not the file's own. The directory prefix stays exactly as the caller spelled it. Three rails keep this from ever *relocating* a path. `fs.realpathSync.native` also resolves symlinks and Windows junctions, so it can answer with a completely different file: the new segment is adopted only when it case-folds equal to the one it replaces, and only when the resolved file's directory case-folds equal to the directory the caller named. And any failure returns the caller's path untouched. `fs.realpathSync` (without `.native`) is useless for this: on Windows it echoes the caller's own spelling back. */
export function canonicalizeIndexPath(rawPath: string): string {
// Every reader addresses a row through `normalizePath` (resolveIndexPath, getFileEntry, querySymbols), so the writer has to mint the key with that same function or the two never meet. Two of its steps rewrite the DIRECTORY prefix -- `expandShortPath` turns a Windows 8.3 segment into its long form, and `normalizeDarwinSystemAlias` turns `/var/...` into `/private/var/...` -- and the writer used to skip both, keeping "exactly as the caller spelled it". On any machine whose temp or project path is reached through an 8.3 alias or the macOS `/var` symlink, that wrote rows under a spelling no lookup can ever produce: the file indexed, and `symbol`, `read`, `refs` and `section` all answered as if it had not. Normalizing the caller's spelling here costs nothing where the path is already canonical, which is the common case and why this went unseen. It still does not adopt realpath's answer for the directory, which is the churn the paragraph above warns about: `normalizePath` rewrites the prefix the same way for every caller, so the spellings converge rather than following whoever happened to call first. The drive letter is the one piece of directory case it touches, and it lowercases it for every caller alike.
const absPath = normalizePath(rawPath)
if (!isCaseInsensitiveFs()) return absPath
let real: string
try {
real = fs.realpathSync.native(absPath)
} catch {
// Unreadable or gone right now -- a git-tracked file deleted from the worktree reaches the indexer on every run. Keep the name callers know it by rather than guess at a spelling.
return absPath
}
// Split on the caller's own string, not a normalized copy: normalizePath lowercases a UNC host and share, so rebuilding the prefix from it would rewrite `//SERVER/Share/...` to `//server/share/...` on the runs that correct a name and leave it alone on the runs that do not -- a second spelling difference introduced by the very function meant to remove one.
const cut = Math.max(absPath.lastIndexOf('/'), absPath.lastIndexOf('\\'))
const base = absPath.slice(cut + 1)
const realNorm = normalizePath(real)
const realBase = path.basename(realNorm)
if (base === realBase) return absPath
if (foldPath(base) !== foldPath(realBase)) return absPath
// The resolved file must live in the directory the caller named. `fs.realpathSync.native` also follows symlinks and Windows junctions, so `alias.ts` can resolve to `../elsewhere/Alias.ts` -- fold-equal basenames, entirely different file. Adopting that spelling would name a directory entry that does not exist where the row says it does.
const callerDir = cut < 0 ? normalizePath(path.resolve('.')) : normalizePath(absPath.slice(0, cut))
if (foldPath(callerDir) !== foldPath(path.dirname(realNorm))) return absPath
return absPath.slice(0, cut + 1) + realBase
}
/** True when the index's stored spelling of `absPath` no longer matches the file's real spelling on disk. Only ever true on a case-insensitive filesystem, which is where the problem lives. {@link getFileEntry} finds a row by folded path, so after `mv b.ts B.ts` the lookup still hits and the content is byte-identical -- the sha gate skips the file, and the row goes on saying `b.ts` forever. Nothing else corrects it: the file still exists, so the deletion sweep leaves it alone, and only a later change to the file's *content* rewrites the row. Until then every `symbol`, `read`, `refs` and `map` answer names the path with a spelling it no longer has, which is the one thing this tool exists to get right. The real spelling is read from the filesystem rather than taken from the caller, because no caller's spelling is authoritative: the walk reports the dirent's, git reports the one in its own index (still the old one until the rename is staged), and the edit hook reports whatever the editor passed. Trusting the caller would let two of them rewrite the row back and forth on alternate runs. Deliberately narrowed to a pure case difference, by the fold comparison against the resolved spelling on the last line. `realpathSync` also resolves symlinks and Windows junctions, so a project reached through a link resolves to a structurally different path -- without that comparison every file in such a project would read as stale on every run and the whole tree would reindex forever. The identical fold check earlier is a cheap filter for a caller naming a structurally different path, not a second correctness guard. This costs one realpath per file per index run, which is noise beside the full-file read and hash `fingerprintFile` already performs for the same file on the same pass. A file whose spelling has actually drifted pays for two more, one in each writer (see canonicalizeIndexPath) -- three in total, on the one run that corrects it and never again. */
export function indexedPathSpellingIsStale(storedPath: string, absPath: string): boolean {
if (!isCaseInsensitiveFs()) return false
// Both sides go through normalizePath, and the stored side is not exempt: the incremental worker writes the dirty queue's own spelling straight through, so a row indexed by the daemon can carry an upper-case drive letter (`C:/...`) where the bulk `index` command's rows carry the lower-case one normalizePath produces. Comparing a raw stored value against a normalized candidate makes that difference look like a stale spelling, and the file then reindexes on every single drain, forever.
const stored = normalizePath(storedPath)
const candidate = normalizePath(path.resolve(absPath))
// Cheap structural filter only. Deliberately NOT `storedPath === candidate`: the caller's own spelling agreeing with the stored one proves nothing, because the caller may simply be repeating the same stale spelling the row already holds -- git reports the name in its own index, which still says `readme.md` until a case rename is staged, and an editor hook reports whatever the editor passed. Short-circuiting on that agreement is how the row would stay stale forever in exactly the cases that never reach a filesystem walk.
if (foldPath(stored) !== foldPath(candidate)) return false
let real: string
try {
real = normalizePath(fs.realpathSync.native(absPath))
} catch {
// Unreadable or gone right now: leave the row alone rather than guess at a spelling.
return false
}
if (real === stored) return false
if (foldPath(real) !== foldPath(stored)) return false
// Both real and stored have the same folded path, but real was canonicalized from the filesystem root by fs.realpathSync.native (expanding canonical case for all parent directories, e.g. C:/Windows/Temp vs C:/WINDOWS/TEMP), whereas stored/candidate may share an ambient directory prefix. Compare segments: if the filename (last segment) differs, the file was renamed.
const storedSegments = stored.split('/')
const realSegments = real.split('/')
if (storedSegments.length !== realSegments.length) return real !== stored
const storedBase = storedSegments[storedSegments.length - 1]
const realBase = realSegments[realSegments.length - 1]
if (storedBase !== realBase) return true
for (let i = storedSegments.length - 2; i >= 0; i--) {
if (storedSegments[i] !== realSegments[i]) {
if (storedSegments.slice(0, i + 1).join('/') === candidate.split('/').slice(0, i + 1).join('/')) {
continue
}
return true
}
}
return false
}
/** The shared read side of the embed-freshness gate used by both worker.ts::makeIndexer and cli.ts's bulk index loop. Returns true when the file's stored `embed_sha` already represents the correct terminal embedding state for the CURRENT environment, so re-running indexFileEmbeddings would do no useful work and can be skipped: - embeddings config-disabled: fresh only when stored is the `disabled:` marker for this sha. - enabled + a bare sha match: fresh (the file was really embedded, or was empty / policy-skipped with nothing to embed -- both are terminal regardless of deps). - enabled but deps currently unavailable: an `unavailable:` marker for this sha is also fresh, so an unchanged file is not re-entered on every worker drain while deps stay missing. - enabled + deps available: an `unavailable:` (or `disabled:`) marker is NOT fresh, forcing the real first embed now that it can finally succeed. - enabled + an `oversize:` marker: fresh only while `symbolOnlyKb` still matches the threshold the marker was stamped under, so raising `indexing.large_file_symbol_only_kb` re-examines the file instead of leaving it permanently skipped. See {@link OVERSIZE_EMBED_SHA_PREFIX}. `symbolOnlyKb` is the caller's current `indexing.large_file_symbol_only_kb`. Pass 0 when no config is in hand: config validation floors that key at 1, so 0 matches no stamped marker and the file is re-examined rather than assumed current, which is the safe direction. */
export function isEmbedFresh(
storedEmbedSha: string | undefined,
sha: string,
embeddingsEnabled: boolean,
depsAvailable: boolean,
symbolOnlyKb: number,
maxChunksPerFile = 0,
): boolean {
if (storedEmbedSha === undefined) return false
if (!embeddingsEnabled) return storedEmbedSha === disabledEmbedSha(sha)
if (storedEmbedSha === sha) return true
if (!depsAvailable && storedEmbedSha === unavailableEmbedSha(sha)) return true
if (storedEmbedSha === oversizeEmbedSha(sha, symbolOnlyKb)) return true
// Same shape as the oversize clause above: threshold-bearing, so a config change re-examines.
if (maxChunksPerFile > 0 && storedEmbedSha === maxChunksEmbedSha(sha, maxChunksPerFile)) return true
// Set-id-bearing rather than ignored, for the reason given on ASSET_EMBED_SHA_PREFIX.
if (storedEmbedSha === assetEmbedSha(sha)) return true
// Not a parameter the way symbolOnlyKb is: the document work clock is a compiled-in constant, not a config value, so there is no caller who could know a different one. It is compared rather than ignored so that changing it in a later release invalidates every stamp taken under the old one.
if (storedEmbedSha === timeoutEmbedSha(sha, MAX_DOCUMENT_WORK_MILLIS)) return true
return false
}
/** `sha`, when provided, is stamped into `files.embed_sha` after {@link embedIndexFile} commits successfully -- tracked separately from `files.sha` (the parse-freshness gate) so a crash or thrown error mid-embedding never gets masked by the parse-sha gate: the embed_sha column is left at its previous (stale/empty) value on any early return or thrown error below, so a later touch of byte-identical content still re-triggers embedding instead of being permanently sha-gate-skipped. See makeIndexer in worker.ts for the read side of this gate. */
export async function indexFileEmbeddings(
rawPath: string,
dbPath: string = globalDbPath(),
sha?: string,
onError?: (err: unknown) => void,
): Promise<void> {
// Same canonicalization as indexFileSync, for the same reason: chunk rows must be keyed by the spelling the parse side wrote, not by whatever the caller happened to pass.
const filePath = canonicalizeIndexPath(rawPath)
const ixCfg = loadConfig().indexing
if (!ixCfg.embeddings_enabled) {
// Stamp a disabled-marker embed_sha even though no embedding actually ran, so makeIndexer's embedUnchanged gate (worker.ts) can hold for this content the next time it's touched while STILL disabled -- otherwise every re-touch of an unchanged file re-enters indexFileEmbeddings just to hit this same early-return again, on every drain, for as long as embeddings stay disabled. Deliberately NOT the real sha (see disabledEmbedSha's doc comment): re-enabling embeddings later must not be mistaken for "already embedded, unchanged".
stampEmbedSha(getDb(dbPath), filePath, sha, disabledEmbedSha)
disabledStamps += 1
return
}
if (filePath.toLowerCase().endsWith('.profile-meta.xml')) {
// Profiles are frequently multi-megabyte, highly repetitive permission dumps. Embedding them creates thousands of low-signal vectors; exact symbol/read/grep access remains.
const db = getDb(dbPath)
deleteFileEmbeddings(db, filePath)
// Deliberately-never-embed is a terminal state: stamp the real sha so the freshness gate (worker.ts/cli.ts) treats this file as done and does not re-read its multi-megabyte content into indexFileEmbeddings on every worker drain / index run.
stampEmbedSha(db, filePath, sha, (s) => s)
return
}
if (isNonTextAsset(filePath)) {
// A JPEG, a font, an archive: encoded bytes with no text in them. Nothing below this point checks that, so before this gate existed these files fell straight through to the generic decodeSource() read at the bottom, which does not fail -- it yields mojibake, and the chunker then cuts that into thousands of tiny control-character chunks, each carrying a 384-dimensional vector that competes for neighbours against real source. Gated on extension rather than on a byte sniff deliberately: the extension is what routed the file here (`token-goat index` enumerates via git ls-files, which returns every tracked file regardless of language, so walkProject's detectLanguageOfFile() !== 'unknown' filter never sees these), and an extension test cannot be fooled by a header that happens to decode. Ordered before the isEmbeddableDocument branch so the two lists can never both claim a path; they are disjoint today and this keeps that a property of the code rather than of the lists.
const db = getDb(dbPath)
deleteFileEmbeddings(db, filePath)
stampEmbedSha(db, filePath, sha, assetEmbedSha)
return
}
if (isEmbeddableDocument(filePath)) {
// Binary document formats (PDF/DOCX/PPTX/XLSX) need format-specific extraction, not a raw utf8 read of the file's bytes -- must run before the generic read below, which would otherwise reinterpret binary content as garbage text. detectLanguage() returns 'unknown' for these extensions (no Language union member, no code symbols), so none of the ipynb/large-file/salesforce branches above or below apply to them.
let extracted: string | null
let refusedOnTheClock = false
try {
extracted = await extractEmbeddableDocumentText(filePath)
} catch (err) {
if (!isDocumentRefusal(err)) {
// Left unstamped on purpose, the same way a thrown embedIndexFile is below: this file's extraction failed rather than declined, and a stamp here would be a permanent verdict recorded from a temporary condition -- pdfjs briefly missing, the file mid-write -- so the document would never be embedded again even once the cause was gone.
onError?.(err)
return
}
refusedOnTheClock = isTransientDocumentRefusal(err)
extracted = null
}
if (extracted === null || extracted.trim().length === 0) {
// A refused document or one with no extractable text is a terminal deliberately-never-embed state, same shape as the .profile-meta.xml skip above: stamp the real sha so an unchanged file is not re-read into extraction on every worker drain / index run. Except when the refusal was the work clock, which is terminal in neither direction: re-reading it every drain would re-spend the whole minute the clock exists to cap, and a bare sha would bury the document forever over one busy minute. The clock-bearing marker is the middle: settled for as long as this clock is, re-examined the moment it moves.
const db = getDb(dbPath)
deleteFileEmbeddings(db, filePath)
stampEmbedSha(db, filePath, sha, (s) => (refusedOnTheClock ? timeoutEmbedSha(s, MAX_DOCUMENT_WORK_MILLIS) : s))
return
}
if (extracted.length > ixCfg.large_file_symbol_only_kb * 1024) {
// Reuse the same large-file threshold as the generic content-length check below -- extracted document text is comparatively expensive to embed for comparatively little retrieval value once it's this large, not a case that needs its own config knob. Stamped with the threshold-bearing oversize marker, not a bare sha, for the reason given on OVERSIZE_EMBED_SHA_PREFIX: this skip is conditional on a user-tunable value, so it is not terminal the way the extraction-failure case above is.
const db = getDb(dbPath)
deleteFileEmbeddings(db, filePath)
stampEmbedSha(db, filePath, sha, (s) => oversizeEmbedSha(s, ixCfg.large_file_symbol_only_kb))
return
}
try {
const db = getDb(dbPath)
// No symbol table exists for these formats, so boundaries is empty -- the whole extracted text goes through chunkFile's generic windowed chunking instead of symbol-aligned chunks.
const outcome = await embedIndexFile(db, filePath, extracted, [], sha)
// The file was reindexed while it embedded; the newer run owns the vectors and the stamp
if (outcome === 'stale') return
// Documents reach this point without having passed the chunk-count gate below, which sits in the generic-content branch this one returns before. A PDF or spreadsheet whose extracted text is under the byte threshold was therefore embedded with no ceiling at all, on the same windowed cut that the ceiling exists to bound. Enforcing it here rather than adding a second pre-count keeps one rule for both branches.
if (outcome !== 'unavailable' && enforceStoredChunkCap(db, filePath, sha, ixCfg.max_chunks_per_file)) return
stampEmbedSha(db, filePath, sha, (s) => (outcome === 'unavailable' ? unavailableEmbedSha(s) : s))
} catch (err) {
onError?.(err)
}
return
}
let content: string
try {
content = decodeSource(await fs.promises.readFile(filePath))
} catch {
return
}
// A dotenv file's values are secret by the file's nature, and this is the one indexing path that persists raw file text: the symbol table already stores env keys with empty bodies, but chunks held the whole file, so `semantic` served the password. Redact before chunking, not at search time, so nothing sensitive is written to disk in the first place. See dotenv_redact.ts.
content = redactIfDotenv(filePath, content)
if (detectLanguage(filePath) === 'ipynb') {
// Embedding boundaries below are line ranges taken from the symbols table, which indexFileSync populated from the SAME virtual document -- must transform content identically here or chunk text would be sliced from the wrong (raw JSON) place. A non-Python-kernel/unparseable notebook is a deliberate terminal never-embed, same shape as the profile-meta.xml/oversized-metadata skips below.
const virtual = ipynbToVirtualSource(content)
if (virtual.cellLanguage === null) {
const db = getDb(dbPath)
deleteFileEmbeddings(db, filePath)
stampEmbedSha(db, filePath, sha, (s) => s)
return
}
content = virtual.content
}
if (content.length > ixCfg.large_file_symbol_only_kb * 1024) {
// Between the symbol-only and full-skip thresholds: syntactic symbols/refs are already indexed by indexFileSync (only large_file_skip_kb gates that), but embedding a moderately-large file is comparatively expensive for comparatively little retrieval value -- skip embedding it. Unlike the profile-meta.xml / salesforce_metadata skips below, this one is conditional on a user-tunable config value rather than on the content, so it is stamped with the threshold-bearing oversize marker instead of a bare sha: see OVERSIZE_EMBED_SHA_PREFIX for what a bare sha cost here.
const db = getDb(dbPath)
deleteFileEmbeddings(db, filePath)
stampEmbedSha(db, filePath, sha, (s) => oversizeEmbedSha(s, ixCfg.large_file_symbol_only_kb))
return
}
if (detectLanguage(filePath) === 'salesforce_metadata' && content.length > 512 * 1024) {
// Keep unusually large generated metadata from producing thousands of low-signal chunks.
const db = getDb(dbPath)
deleteFileEmbeddings(db, filePath)
// As with the profile skip above, this is a terminal deliberately-never-embed state; stamp the real sha so re-touching the (unchanged) file does not re-enter this path every drain.
stampEmbedSha(db, filePath, sha, (s) => s)
return
}
const boundaries = buildEmbeddingBoundaries(filePath, content, dbPath)
// Count what this file would contribute before embedding any of it. Byte size is already gated above and does not catch this case: chunk cuts snap to structure, so a generated data snapshot of a few hundred kilobytes -- thousands of one-line keys, each its own boundary -- becomes thousands of near-identical chunks, while a source file of the same size becomes a few hundred meaningful ones. On one real index the worst 32 files were nearly all under the byte threshold and held 22% of every chunk on the machine. This cut is pure string work against content already in memory and costs nothing next to the model inference this gate exists to avoid; it is deliberately NOT hoisted into embeddings.ts, because that file is hashed into EMBED_FINGERPRINT and editing it would force every already-embedded file on every machine to be re-embedded. It is a LOWER bound, not the count that will be stored. embedIndexFile cuts again with the model's tokenizer whenever the embedding deps are present, and that cut is strictly finer, so clearing the ceiling here does not mean the rows will. enforceStoredChunkCap re-applies the ceiling to what actually landed; this stays as the free rejection of the files that are already over it on the coarse cut, which saves their inference entirely.
const chunkCount = chunkFile(filePath, content, undefined, undefined, boundaries).length
if (chunkCount > ixCfg.max_chunks_per_file) {
const db = getDb(dbPath)
deleteFileEmbeddings(db, filePath)
stampEmbedSha(db, filePath, sha, (s) => maxChunksEmbedSha(s, ixCfg.max_chunks_per_file))
return
}
try {
const db = getDb(dbPath)
const outcome = await embedIndexFile(db, filePath, content, boundaries, sha)
// The file was reindexed while it embedded; the newer run owns the vectors and the stamp
if (outcome === 'stale') return
if (outcome !== 'unavailable' && enforceStoredChunkCap(db, filePath, sha, ixCfg.max_chunks_per_file)) return
// When the optional embedding deps were absent, embedIndexFile reports 'unavailable' and no vectors were written -- stamp an unavailable-marker embed_sha (not the bare sha) so this file is re-embedded once the deps are installed, rather than masquerading as fresh forever.
stampEmbedSha(db, filePath, sha, (s) => (outcome === 'unavailable' ? unavailableEmbedSha(s) : s))
} catch (err) {
// Best-effort: never fail the overall index over an embeddings-only error. embed_sha is deliberately left unstamped here (see doc comment above). `onError`, when provided, lets a caller (worker.ts's embedFileSerialized) record this failure somewhere discoverable -- this function itself never throws, matching its documented best-effort contract for callers like cli.ts's foreground bulk-index loop that await it directly with no try/catch.
onError?.(err)
}
}
/** Stamp `files.embed_sha` for `filePath`, but only when a `sha` was actually provided (the incremental worker/CLI paths always pass one; some callers do not). `makeValue` derives the value to store from the (now-defined) sha -- identity for a real/terminal embed, or a disabled:/unavailable: marker. Centralizes the UPDATE so every early-return in {@link indexFileEmbeddings} records its terminal state identically. */
function stampEmbedSha(
db: ReturnType<typeof getDb>,
filePath: string,
sha: string | undefined,
makeValue: (sha: string) => string,
): void {
if (sha === undefined) return
// Optimistic-concurrency guard: also require files.sha to still equal the sha this embed run started from. inFlightEmbeddings (worker.ts) only serializes concurrent embed calls WITHIN a single process -- it cannot see a second process (e.g. a slow foreground `token-goat index` racing the background daemon) embedding the same file at the same time. Without this WHERE clause, a slow writer that started against an older `sha` can still commit its stamp AFTER a faster writer already reindexed and re-embedded a newer version, overwriting the fresher embed_sha with a stale one and leaving embeddings silently out of sync with no way to detect it. Requiring sha = ? makes a stale writer's stamp a no-op instead: the row's `sha` will have already moved on to the newer value by the time the stale writer's UPDATE runs.
db.prepare(`UPDATE files SET embed_sha = ? WHERE ${pathEqClause('path')} AND sha = ?`).run(
makeValue(sha),
foldPath(filePath),
sha,
)
}
function safeMtime(filePath: string): number {
try {
return fs.statSync(filePath).mtimeMs / 1000
} catch {
return 0
}
}