Repository navigation
Expand file tree
/
Copy pathworker.ts
More file actions
1032 lines (960 loc) · 99.4 KB
/
Copy pathworker.ts
File metadata and controls
1032 lines (960 loc) · 99.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/** Background worker: drain the dirty queue and re-index changed files. Ports the daemon loop of `worker.py` to the TypeScript surface. The loop reads `{dataDir}/queue/dirty.txt` (the format lives in dirty_queue.ts), parses each changed path, and writes its symbol/ref rows into the index DB via {@link indexFileSync}. Processed entries are cleared from the queue before sleeping `pollIntervalMs`. Spawning the detached daemon that runs this loop, and finding or stopping it through its pid file, lives in worker_lifecycle.ts, so a hook that only needs the daemon alive never loads the drain. */
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { dataDir, globalDbPath } from './constants.js'
import { fileIsAbsent, fingerprintFileWithBytes, indexedFileIsGone } from './fingerprint.js'
import { assetEmbedSha, disabledStampCount, indexFileSync, indexFileEmbeddings, indexedPathSpellingIsStale, isEmbedFresh, isParseSkipEligible, loadRegexExtractors, maxChunksEmbedSha } from './parser.js'
import { parserFingerprintForLanguage } from './parser_stamp.js'
import { deleteFileEmbeddings, embeddingsDepsAvailable, ensureEmbeddingProvenance } from './embeddings.js'
import { pruneUnembeddableChunks } from './embed_backfill.js'
import { getFileEntry } from './index_reader.js'
import { readPreHashStamp, restampVerifiedUnchanged, type PreHashStamp } from './reconcile.js'
import { normalizePath } from './paths.js'
import { countNoun, foldPath, isUnderBlockedRoot, extractErrorMessage } from './util.js'
import { configProjectRootFor, loadConfig, withConfigProjectRoot } from './config.js'
import { getDb } from './db.js'
import { pathEqClause } from './sql_path.js'
import { removeFileFromIndex, pruneDeletedFiles, sweepKnownRoots } from './index_prune.js'
import { recordKnownRootThrottled, sweepExpiredKnownRootMarkers } from './known_roots.js'
import { reclaimIndex, walCheckpointBusyNoWait } from './index_reclaim.js'
import { applyIndexingPriority } from './process_priority.js'
import { cleanup_stale } from './snapshots.js'
import { sweepCacheRoots } from './disk_cache.js'
import { sessionsDir } from './sessions_dir.js'
import { sweepStaleShrinkCopies } from './shrink_temp_copies.js'
import { findProject, isUnderSystemTemp } from './project.js'
import { registerReset } from './reset.js'
import { modelDownloadHeld } from './embed_preflight.js'
import { failedAtOf } from './model_download_gate.js'
import { appendDirtyQueueFile, appendDirtyQueuePaths, dirtyQueuePathFor, parseDirtyQueueLines } from './dirty_queue.js'
import { createQueueWaker, noteOwnQueueWrite } from './queue_waker.js'
import { drainReconcileRequestsAsync } from './reconcile_requests.js'
import { appendWorkerErrorLog, claimCurrentWorker, readPidFile, resolvePollIntervalMs, retireDisplacedWorker, WORKER_STARTUP_GRACE_MS, workerErrorLogPath, workerPidPath, writeDrainHeartbeat } from './worker_lifecycle.js'
// Stale session-snapshot sweep runs on the same loop as the dirty-queue drain (see runWorkerLoop) but throttled to this interval -- cleanup_stale's own default 24h staleness window doesn't need finer-grained sweeping than hourly, and a full directory scan on every 2s poll tick would be wasteful.
const SNAPSHOT_CLEANUP_INTERVAL_MS = 60 * 60 * 1000
/** How long a foreground session write suppresses heavy background worker maintenance (embedding backlog walks, known-roots sweeps, snapshot/cache sweeps, idle vacuuming) so interactive CLI turns do not compete with background I/O. */
export const FOREGROUND_ACTIVITY_YIELD_MS = 6_000
/** True when any session state in `sessDir` (defaults to {@link sessionsDir}) was updated within {@link FOREGROUND_ACTIVITY_YIELD_MS} of `now`. Absent or unreadable directories return false so idle daemons and isolated tests proceed normally. */
export function hasRecentForegroundActivity(now = Date.now(), sessDir: string = sessionsDir()): boolean {
try {
const mtimeMs = fs.statSync(sessDir).mtimeMs
return now - mtimeMs >= 0 && now - mtimeMs < FOREGROUND_ACTIVITY_YIELD_MS
} catch {
return false
}
}
/** How many {@link drainOnce} cycles to skip between opportunistic {@link pruneDeletedFiles} sweeps. A `git mv`, directory rename, `git checkout <branch>`, or `git clean` never fires an Edit hook for the paths it touches, so a plain dirty-queue drain (which only reconciles a deletion when the exact old path was enqueued) never notices those rows are gone and they orphan in the index forever. `pruneDeletedFiles` walks every indexed path under a project root and stats each one to catch this — too expensive to repeat on every ~2s poll tick, so it only runs every Nth cycle (mirrors SNAPSHOT_CLEANUP_INTERVAL_MS's throttling above, keyed on drain-cycle count here instead of wall time since the trigger is "enough drain activity", not "enough time elapsed"). */
const PRUNE_EVERY_N_DRAINS = 30
/** How often a drain in progress re-reads the pid file to see whether its daemon must retire. A drain runs synchronously and can take far longer than the startup grace, so the retirement check has to happen inside it, not only between cycles; this bounds the reads to a few per second however many paths the batch holds. */
const RETIRE_PROBE_INTERVAL_MS = 100
/** The retirement check of the daemon loop running in this process, installed by {@link runWorkerLoop} for as long as the loop runs. Null outside a daemon loop, where nothing retires. Process state rather than a parameter: one daemon runs one loop, and a drain reaches the check through the call chain it already has. */
let retireCheckForDrain: (() => boolean) | null = null
/** True once the daemon running this drain has started retiring: a drain then hands its unreached paths back to the live queue (no retry is charged, since nothing failed) and claims nothing new. */
function drainShouldHandBack(): boolean {
return retireCheckForDrain?.() ?? false
}
/** Per-dataDir drain-cycle counters, so unrelated worker instances (distinct test dirs, or a future multi-dataDir setup) don't share a single global cadence. */
const drainCycleCounts = new Map<string, number>()
/** Per-dataDir last-known project root, opportunistically learned from the dirty paths {@link processDirtyBatch} actually processes. The dirty queue is global and path-keyed (one shared `dataDir`, not one per project — see the module doc comment), so there is no standing notion of "the active project" for the periodic prune sweep to target; the files currently flowing through the queue are the closest available signal for which project is active. This is deliberately best-effort: a rename/delete that happens without any other Edit-hook traffic in between won't be pruned until the NEXT unrelated edit in that project re-establishes the project root, which is an acceptable trade-off for orphaned rows that would otherwise never be cleaned up at all. */
const lastKnownProjectRoots = new Map<string, string>()
/** Tracks '.draining' files whose content was already folded into a batch by {@link drainOnce} but could not be removed or quarantined (both cleanup attempts failed, e.g. a persistent Windows sharing violation). Keyed by the '.draining' file's absolute path, valued by the exact content already processed. Without this, a `.draining` file that survives a full drain cycle unchanged would be re-read and its paths reprocessed on every subsequent cycle until cleanup finally succeeds. */
const unclearedDrainingSnapshots = new Map<string, string>()
/** Identity-plus-content stamp for a draining file, used as the {@link unclearedDrainingSnapshots} value. Content alone is not enough to decide "we already folded this file in": a *different* file reusing the same `.draining` name can hold byte-identical content, which is the common case rather than a rare one, since re-editing the same source file queues the same path again. A stale snapshot matching that way makes stage (a) skip a batch nobody processed -- reachable when a crash between stage (b)'s claim-rename and its batch leaves the claimed file unprocessed. mtime and size separate the two: cycles are seconds apart, so a recreated file differs. */
function drainingSnapshotStamp(file: string, content: string): string {
let identity = 'unknown'
try {
const stat = fs.statSync(file)
identity = `${stat.mtimeMs}:${stat.size}:${stat.ino}`
} catch {
// Unstattable: fall back to content-only matching, which is the previous behavior.
}
return `${identity}::${content}`
}
/** How many times stage (a) re-reads a `.draining` file before treating it as genuinely unreadable. Mirrors stage (b)'s claim-rename retry budget. */
const DRAINING_READ_ATTEMPTS = 5
/** Delay between the stage (a) read retries above, in milliseconds. */
const DRAINING_READ_RETRY_DELAY_MS = 50
/** List every live draining-recovery file for `queuePath`: the primary `dirty.txt.draining` name, plus any `.alt-<ts>` fallback claimed by stage (b) when the primary name was still occupied by a file a previous cycle could not clean up (see drainOnce's stage (b) comment). Excludes `.corrupt-*` quarantine files, which are deliberately abandoned and must never be reprocessed. Without recovering every fallback file (not just the first), a single stuck primary `.draining` file could starve the rest of the queue from ever draining. */
function listDrainingFiles(queuePath: string): string[] {
const dir = path.dirname(queuePath)
const base = `${path.basename(queuePath)}.draining`
let entries: string[]
try {
entries = fs.readdirSync(dir)
} catch {
return []
}
return entries
.filter((name) => name === base || (name.startsWith(`${base}.alt-`) && !name.includes('.corrupt-')))
.sort()
.map((name) => path.join(dir, name))
}
/** Cap on consecutive transient-read-failure requeues for the same path (see {@link requeueDirtyPath}). Without this, a file with a permanently stuck read lock (or any other per-file failure that never clears) gets requeued forever, every ~2s drain cycle, with no cap and no throttled visibility into the fact that it's stuck. */
const MAX_TRANSIENT_RETRIES = 5
/** Read and increment `index_retries.retry_count` for `absPath` in the index DB at `dbPath`, creating the row if the path has no streak yet. Returns the count AFTER incrementing. The counter lives in its own table rather than in `files` because a path that has never been indexed has no `files` row: minting one here to hold the counter made a `files` row stop meaning "this file is in the index", and `indexMatchesDisk`, `healStaleIndex` and `staleWarning` all read such a row as a legacy pre-fingerprinting entry and accept it as-is. See the `index_retries` comment in db.ts. Persisted in the index DB rather than an in-memory Map so the count survives across the hook-process/daemon-process boundary -- see {@link clearRetryCount}'s doc comment for the cross-process bug this closes. Matched by the same folded-path convention as every other file_path/path comparison in this codebase (see {@link foldPath}, {@link pathEqClause}), so a case-variant reference to the same file on a case-insensitive filesystem shares one counter. Wrapped in a transaction so the read-then-write is atomic against a concurrent writer (the daemon and a CLI hook process both have DB access). */
function bumpRetryCount(dbPath: string, absPath: string): number {
const db = getDb(dbPath)
// The files.path column stores paths normalized by normalizePath() (forward slashes) -- every real writer (hooks_edit.ts's postEditHandler, cli.ts's cmdIndex via resolveIndexPath) normalizes before the path ever reaches the queue or the indexer; see sql_path.ts's projectScopeClause doc for the same invariant stated from the SQL side. Normalize defensively here too so correctness doesn't silently depend on every future caller remembering to pre-normalize -- a caller that passed a native backslash path straight through used to always miss the SELECT and fall into the INSERT branch below instead of matching the existing row (see the regression test in worker.test.ts).
const normalized = normalizePath(absPath)
const folded = foldPath(normalized)
const tx = db.transaction((): number => {
const row = db.prepare(`SELECT retry_count FROM index_retries WHERE ${pathEqClause('path')}`).get(folded) as
| { retry_count: number | null }
| undefined
if (row !== undefined) {
const next = (row.retry_count ?? 0) + 1
db.prepare(`UPDATE index_retries SET retry_count = ? WHERE ${pathEqClause('path')}`).run(next, folded)
return next
}
db.prepare('INSERT INTO index_retries (path, retry_count) VALUES (?, 1)').run(normalized)
return 1
})
// `.immediate()` -- BEGIN IMMEDIATE. The driver issues a plain call as a deferred BEGIN, which takes a read snapshot first and only asks for the write lock at the first writing statement. SQLite refuses that upgrade with SQLITE_BUSY straight away instead of consulting the busy handler, so `busy_timeout` does nothing for it and a concurrent writer fails outright. This database is shared by the worker daemon, the hook processes and the CLI at once, so that is an ordinary situation rather than a rare one. See writeParseResult in parser.ts.
return tx.immediate()
}
/** Drop `absPath`'s transient-failure streak in the index DB at `dbPath`. Best-effort: a DB error here (e.g. the DB does not exist yet) must not block the caller's own already-completed work. No-op if the path has no streak -- nothing to reset. Called from {@link processDirtyBatch} for every path whose `fingerprintFileWithBytes` read succeeds during a drain, so a path that built up a retry streak during a transient lock episode (an antivirus scan, an editor holding the file, a OneDrive sync) starts from a full budget again the moment it can actually be read. The count is kept in the index DB (`index_retries`) rather than an in-memory Map because the processes involved do not share a heap: the edit hook runs in a short-lived CLI process while the drain loop that reads the count runs in the long-lived detached daemon. A reset that only mutated a module-level Map would be invisible to the daemon's own copy of it and would silently do nothing in the real deployed topology -- an exhausted path would stay permanently given-up-on. The index DB is the one thing both processes already share, as they do for `files.sha` and `files.embed_sha`. `appendDirtyPath` (`hooks_index.ts`) deliberately does NOT call this on the edit-hook path; its comment there explains why, and the drain-time reset above is what covers that case. */
export function clearRetryCount(dbPath: string, absPath: string): void {
try {
const db = getDb(dbPath)
// See bumpRetryCount's doc comment: normalize defensively to match the normalized form the row was written under.
const folded = foldPath(normalizePath(absPath))
db.prepare(`DELETE FROM index_retries WHERE ${pathEqClause('path')}`).run(folded)
} catch {
// best-effort -- see doc comment above.
}
}
/** Cap on worker-errors.log's size before {@link cleanupWorkerStateFiles} rotates (truncates) it. Mirrors disk_cache.ts's pruneBlobs size/age-cutoff pattern for keeping other accumulating state bounded over a long-lived daemon's lifetime -- nothing previously rotated this file, so it could otherwise grow unbounded across a project's entire index lifetime. */
const WORKER_ERROR_LOG_MAX_BYTES = 5 * 1024 * 1024
/** How old a `.draining.corrupt-<timestamp>` quarantine file (see drainOnce's cleanup-failure fallback) must be before {@link cleanupWorkerStateFiles} removes it. Mirrors disk_cache.ts's pruneBlobs age cutoff and snapshots.ts's cleanup_stale 24h window, scaled up: a quarantine file is kept around long enough to be manually inspected after a persistent lock/corruption issue, not treated as routine cache churn. */
const CORRUPT_QUARANTINE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000
/** Best-effort housekeeping for the worker's own accumulating state files under `dir`: - rotates (truncates) `worker-errors.log` once it exceeds {@link WORKER_ERROR_LOG_MAX_BYTES}. - removes `.corrupt-*` dirty-queue quarantine files older than {@link CORRUPT_QUARANTINE_MAX_AGE_MS}. - removes expired `known-root-record-*.marker` throttle files (see {@link sweepExpiredKnownRootMarkers}). None had any rotation/cleanup before this, so all could grow unbounded over a project's index lifetime. Mirrors the size/age-cutoff cleanup pattern already used elsewhere in this codebase for other accumulating state (disk_cache.ts's pruneBlobs, snapshots.ts's cleanup_stale). Fail-soft: never throws. Called on the same periodic sweep as cleanup_stale in {@link runWorkerLoop}; exported so tests can drive it directly without waiting on real time. */
export function cleanupWorkerStateFiles(dir: string): void {
try {
const logPath = workerErrorLogPath(dir)
const stat = fs.statSync(logPath)
if (stat.size > WORKER_ERROR_LOG_MAX_BYTES) {
fs.writeFileSync(
logPath,
`${new Date().toISOString()} worker-errors.log rotated (exceeded ${WORKER_ERROR_LOG_MAX_BYTES} bytes)\n`,
)
}
} catch {
// Missing log file, or a stat/write failure -- nothing to rotate.
}
try {
const queueDir = path.dirname(dirtyQueuePathFor(dir))
const cutoff = Date.now() - CORRUPT_QUARANTINE_MAX_AGE_MS
for (const file of fs.readdirSync(queueDir)) {
if (!file.includes('.corrupt-')) continue
const full = path.join(queueDir, file)
try {
if (fs.statSync(full).mtimeMs < cutoff) fs.unlinkSync(full)
} catch {
// best-effort per-file cleanup -- one bad quarantine file must not abort the sweep.
}
}
} catch {
// Missing queue dir, or a readdir failure -- nothing to clean.
}
// Third accumulating state file, in the data dir rather than the queue dir: one throttle marker per directory ever edited, with no reaper of its own.
sweepExpiredKnownRootMarkers(dir)
}
/** Sentinel returned by {@link makeIndexer}'s default callback when `indexFileSync` (or the sha-gate lookup preceding it) throws. Distinct from the sha-gate's own `false` no-op-skip return so {@link processDirtyBatch} never conflates "nothing needed reindexing" with "indexing was attempted and failed" -- both must be excluded from the indexed count, but only the latter is a real problem worth logging. */
const INDEX_FAILED = Symbol('indexFailed')
/** Chains concurrent {@link indexFileEmbeddings} calls for the same file so they resolve in submission order rather than completion order. makeIndexer's default callback fires embedding off without awaiting it (see its doc comment -- the drain loop must return instantly), so two drains of a rapidly re-edited file can spawn two concurrent `indexFileEmbeddings` promises for the same path. Without serialization, if the OLDER call (started first, with now-stale content) happens to finish AFTER the newer one, its stale chunks/vectors silently overwrite the fresher ones -- a last-writer-wins bug that nothing self-corrects until the next edit touches the file again. Keyed by the case-folded path (matching {@link foldPath}'s convention used everywhere else in this file for path identity) so same-file concurrency is serialized while different files still embed in parallel. Cleared once a chain settles and nothing newer has been chained onto it, so this map does not grow unbounded over a long-lived daemon's lifetime. */
const inFlightEmbeddings = new Map<string, Promise<unknown>>()
/** Resolve once every embedding call currently tracked in {@link inFlightEmbeddings} has settled -- including one still waiting on the global concurrency slot (see {@link embedFileSerialized}), since a slot's entry is added to the map at dispatch time regardless of whether it started immediately or was queued behind the cap. makeIndexer fires embedding fire-and-forget by design (the drain loop must return instantly -- see its doc comment), so a caller that needs embeddings to have actually completed before proceeding (e.g. a test asserting on embed_sha, or one that closes the index DB right after draining and would otherwise race a queued embed call that hasn't even opened its DB connection yet) can await this instead of guessing at timing. */
export function pendingEmbeddings(): Promise<unknown> {
return Promise.allSettled([...inFlightEmbeddings.values()])
}
/** Global concurrency gate for embedding pipelines across ALL files, honoring `config.worker.max_pool_workers`. {@link inFlightEmbeddings} only serializes duplicate work on the SAME file -- a dirty batch of N distinct changed files previously fired N concurrent `indexFileEmbeddings` transformer-inference pipelines with no cap at all, which can spike CPU/memory proportionally to batch size. `activeEmbedSlots` tracks how many pipelines are currently running; `embedSlotWaiters` holds resolvers for callers queued behind the cap, woken one at a time (FIFO) as slots free up in {@link makeReleaseEmbedSlot}. */
let activeEmbedSlots = 0
const embedSlotWaiters: Array<{ start: () => void; abandon: () => void }> = []
/** Settles every embed still queued for a slot without running it, for a daemon that is retiring. The exit waits (bounded) for embeds to settle, and a drain of thousands of files queues thousands behind the few that run, so waiting for the whole queue spent the entire settle bound after a retirement that had already waited out the startup grace. A file whose embed never started keeps its NULL embed_sha, which the backlog sweep embeds later, so nothing is lost; embeds already running are left to finish. */
export function abandonQueuedEmbeds(): void {
for (const waiter of embedSlotWaiters.splice(0)) waiter.abandon()
}
/** Bumped by the reset below. A release closure carries the epoch it was created under, so an embed that was already running when the reset fired cannot decrement the counter the reset just zeroed. */
let embedSlotEpoch = 0
/** Release a slot claimed inline in {@link embedFileSerialized}, waking the next queued waiter. */
function makeReleaseEmbedSlot(): () => void {
const epoch = embedSlotEpoch
return () => {
// A pre-reset embed settling after the reset used to decrement the fresh counter, taking it negative -- and a negative count means `activeEmbedSlots < limit` stays true for one extra caller per stale task, so the global cap this gate exists to enforce was quietly loosened for the rest of the process. The dispatch it belonged to is gone; its release is too.
if (epoch !== embedSlotEpoch) return
activeEmbedSlots -= 1
embedSlotWaiters.shift()?.start()
}
}
// This counter is intentionally a single process-wide value (not scoped per-file or per-dir like inFlightEmbeddings/lastKnownProjectRoots above) since the concurrency cap it enforces is a real, global daemon-lifetime resource limit. That also means it can leak across test cases that dispatch a real (unresolved-by-test-end) embed call without awaiting it to settle -- register it with the shared reset registry so tests can restore a clean slate via clearModuleCaches() rather than each test needing its own bespoke workaround.
registerReset(() => {
embedSlotEpoch += 1
activeEmbedSlots = 0
embedSlotWaiters.length = 0
// inFlightEmbeddings has to go with them. A call that arrived while the cap was full returns a promise whose only way to settle is the closure sitting in embedSlotWaiters, so dropping that array orphans the promise for good: its `.finally` never runs, its map entry never clears, and pendingEmbeddings() -- which waits on exactly those values -- never resolves. Clearing the map is what makes the "clean slate" above true rather than only true for the counter.
inFlightEmbeddings.clear()
lastLoggedDownloadFailure.clear()
})
/** The recorded model-download failure each data dir last logged, by its timestamp. Every embed queued before the hold took effect fails with the same recorded failure, and logging each one buried the single line that says why under dozens of copies. */
const lastLoggedDownloadFailure = new Map<string, number>()
/** Run {@link indexFileEmbeddings} for `absPath`, serialized against any other in-flight embed call for the same path (see {@link inFlightEmbeddings}) AND capped globally across all files by {@link acquireEmbedSlot}/{@link makeReleaseEmbedSlot}. Errors are swallowed (mirrors the `.catch(() => undefined)` the direct call used before this wrapper existed) so one failed embed never breaks the chain for the next caller. `configRoot` is the file's own project ({@link embedPolicyResolver}), applied at dispatch rather than here because a queued call dispatches after this function has returned. */
function embedFileSerialized(absPath: string, dbPath: string, sha: string, configRoot: string): Promise<unknown> {
const key = foldPath(absPath)
const prior = inFlightEmbeddings.get(key)
const dir = path.dirname(dbPath)
// Unlike a parse failure (logged via logIndexFailure/INDEX_FAILED above), indexFileEmbeddings swallows its own errors internally with no log path at all -- and the background daemon runs with stdio: 'ignore' (see startDetachedWorker), so a thrown embedding error previously produced zero observable trace anywhere. Route it through the same worker-errors.log appendWorkerErrorLog uses for indexing failures so it is at least discoverable after the fact, instead of vanishing silently.
const onEmbedError = (err: unknown): void => {
const failedAt = failedAtOf(err)
if (failedAt !== null) {
if (lastLoggedDownloadFailure.get(dir) === failedAt) return
lastLoggedDownloadFailure.set(dir, failedAt)
}
const message = extractErrorMessage(err)
appendWorkerErrorLog(dir, `${new Date().toISOString()} indexFileEmbeddings failed for ${absPath}: ${message}\n`)
}
// Defensive fallback (matches config.ts's own default of 4): several existing tests in this file mock loadConfig() with only a partial `{ worker: { blocked_roots: [...] } }` shape (see makeIndexer's embeddingsEnabled comment above for the same pattern), so a bare `.max_pool_workers` here would be `undefined` for those and silently block every embed call forever (0 < undefined is false, so nothing would ever dispatch or queue). loadConfig() always returns a fully-populated, schema-validated config object in production.
const limit = loadConfig().worker.max_pool_workers ?? 4
// Dispatches the actual embed call once a global slot is available, releasing it as soon as that call settles (success or failure) so it frees up for the next queued file regardless of outcome. Release is attached as a sibling `.then(release, release)` on the SAME promise `indexFileEmbeddings` returns (rather than wrapped via `.finally()` into a new promise that `runEmbed` then returns) so it costs no extra microtask hop on the chain the rest of this function builds on top of `runEmbed()`'s return value -- preserving indexFileEmbeddings' previous direct-call timing (including for tests that assert on it without awaiting extra ticks) for the common case where a slot is immediately free. Only when the global cap is already saturated does this queue in embedSlotWaiters, deferring dispatch until a running embed elsewhere finishes and calls its release closure.
const dispatchEmbed = (): Promise<unknown> => {
// makeIndexer checks the hold when it queues a file, but a drain queues every file before the first embed runs, so the embeds behind a failed download were still dispatched: each announced "Downloading the embedding model" and then refused. Checking again here, when the embed actually starts (a queued one starts from embedSlotWaiters, not runEmbed), skips them; the file keeps a NULL embed_sha for the backlog sweep. The slot is taken already, so hand it back.
if (modelDownloadHeld()) {
makeReleaseEmbedSlot()()
return Promise.resolve(undefined)
}
// indexFileEmbeddings reads `indexing.embeddings_enabled` with no root to pass, as its first synchronous statement, so the file's project has to be the one loadConfig() resolves for that stretch.
const result = withConfigProjectRoot(configRoot, () => indexFileEmbeddings(absPath, dbPath, sha, onEmbedError))
const release = makeReleaseEmbedSlot()
result.then(release, release)
return result
}
const runEmbed = (): Promise<unknown> => {
if (activeEmbedSlots < limit) {
activeEmbedSlots += 1
return dispatchEmbed()
}
return new Promise<unknown>((resolve) => {
embedSlotWaiters.push({
start: () => {
activeEmbedSlots += 1
resolve(dispatchEmbed())
},
abandon: () => resolve(undefined),
})
})
}
const chained = (prior === undefined ? runEmbed() : prior.then(runEmbed)).catch(() => undefined)
inFlightEmbeddings.set(key, chained)
void chained.finally(() => {
// Only clear the slot if nothing newer has been chained onto it since -- otherwise this `finally` (from an OLDER call resolving after a NEWER one already replaced the map entry) would delete the newer call's still-in-flight tracking.
if (inFlightEmbeddings.get(key) === chained) inFlightEmbeddings.delete(key)
})
return chained
}
function logIndexFailure(dir: string, absPath: string, err: unknown): void {
const message = extractErrorMessage(err)
appendWorkerErrorLog(dir, `${new Date().toISOString()} indexing failed for ${absPath}: ${message}\n`)
}
// A dirty path that exists (fs.existsSync true) but whose fingerprintFileWithBytes call returns null is a transient read failure -- a lock held by an AV scanner/editor/OneDrive sync, a permission error, or a race with an external writer -- not a permanent parse/index error. Before this fix, processDirtyBatch silently `continue`d past it: the path was dropped from this batch and, once drainOnce unconditionally clears the .draining marker right after processDirtyBatch returns, it was gone for good with no log entry and no way to retry it short of the file being touched again. Log it distinctly from an indexing failure and requeue it so the next drain cycle gets another chance once the lock clears.
function logTransientReadFailure(dir: string, absPath: string): void {
appendWorkerErrorLog(
dir,
`${new Date().toISOString()} fingerprintFileWithBytes returned null for existing file ${absPath} (transient read failure -- requeued for retry)\n`,
)
}
// Bumps the transient-retry count for `absPath` and decides whether it is still within its retry budget. Returns true when the caller should go on to actually requeue the path (append it to a dirty queue); false once MAX_TRANSIENT_RETRIES has been exceeded (logging the give-up message exactly once, on the cycle the cap is first exceeded). Split out of the old requeueDirtyPath so drainOnce can bump the retry counter immediately (once per real failure) while deferring the actual queue-file append -- see drainOnce's stage (a)/(b) doc comment for why the append must not land in the SAME cycle's live queue.
function bumpAndCheckRetry(dir: string, absPath: string): boolean {
const dbPath = path.join(dir, 'global.db')
const attempts = bumpRetryCount(dbPath, absPath)
if (attempts > MAX_TRANSIENT_RETRIES) {
// Permanently stuck (e.g. a read lock that never clears): stop requeuing so this path doesn't get hammered every single drain cycle forever. Log exactly once -- on the cycle the cap is first exceeded, not on every subsequent cycle -- so the failure is visible without spamming the log. A future edit to this path re-dirties it through the normal queue-append path (not this function), which gets it one more attempt -- and a fresh budget only once that attempt's read actually succeeds, since clearRetryCount runs on success and the edit hook deliberately does not reset the counter itself (see hooks_index.ts). An edit to a file whose lock has still not cleared therefore gets exactly that one attempt, not MAX_TRANSIENT_RETRIES more.
if (attempts === MAX_TRANSIENT_RETRIES + 1) {
appendWorkerErrorLog(
dir,
`${new Date().toISOString()} giving up on ${absPath} after ${MAX_TRANSIENT_RETRIES} consecutive transient failures -- no longer retrying automatically (will retry again if the file changes)\n`,
)
}
return false
}
return true
}
// The worker's own append to the queue under `dir`, the one processDirtyBatch/drainOnce were given (an isolated dir under test). Best-effort: if the write fails, the path is lost for this cycle, but the failure is still captured via logTransientReadFailure above. Returns whether the paths reached the queue, for a caller that can keep them and try again.
function appendToDirtyQueue(dir: string, ...absPaths: string[]): boolean {
try {
const appended = appendDirtyQueuePaths(dir, absPaths)
if (appended) noteOwnQueueWrite(dir)
return appended
} catch {
return false
}
}
// Re-adds `absPath` to the live dirty queue after a transient read failure. This is the default `requeue` callback processDirtyBatch uses outside of drainOnce (e.g. direct callers/tests); drainOnce itself injects a callback that defers the appendToDirtyQueue half -- see its doc comment for why.
function requeueDirtyPath(dir: string, absPath: string): void {
if (bumpAndCheckRetry(dir, absPath)) appendToDirtyQueue(dir, absPath)
}
/** Answers, for a file, which project's configuration decides whether it is embedded and what that configuration decides: `indexing.embeddings_enabled` from the `.token-goat.toml` that {@link configProjectRootFor} picks for the file's directory. Never the working directory's: the daemon runs in the temp directory and `token-goat index` runs wherever it was typed, and deciding by the working directory made the index and the drain stamp one file two ways, each undoing the other. The drain, the backlog sweep and cmdIndex all decide through one of these, built once per drain, sweep cycle or index run: each directory is walked once, since every level of the walk probes nine project markers, and each root's configuration is read once, and a `.token-goat.toml` added, removed or edited is seen by the next one. */
export function embedPolicyResolver(): (absPath: string) => { configRoot: string; embeddingsEnabled: boolean } {
const roots = new Map<string, string>()
const enabled = new Map<string, boolean>()
return (absPath) => {
const dir = path.dirname(absPath)
let configRoot = roots.get(dir)
if (configRoot === undefined) {
configRoot = configProjectRootFor(dir)
roots.set(dir, configRoot)
}
let embeddingsEnabled = enabled.get(configRoot)
if (embeddingsEnabled === undefined) {
// Partial-config defensiveness for tests that mock loadConfig with a `{ worker: {...} }` shape: default to enabled, config.ts's own default.
embeddingsEnabled = loadConfig(configRoot).indexing?.embeddings_enabled ?? true
enabled.set(configRoot, embeddingsEnabled)
}
return { configRoot, embeddingsEnabled }
}
}
/** Build the index callback the drain loop uses by default: parse each changed file and write its symbol/ref rows into the index DB at `dbPath`. A parse or read failure on one file is swallowed so a single bad file never aborts the batch or crashes the drain loop -- but the failure is logged to {@link workerErrorLogPath} and signalled via the {@link INDEX_FAILED} sentinel so `processDirtyBatch` never counts it as a successful index. The file's `files.sha` row is left exactly as it was before this attempt, so if the file is ever touched again the sha-gate below will not match its (still un-indexed) content and a reindex will be retried automatically. */
export function makeIndexer(dbPath: string): (absPath: string, sha: string, stamp?: PreHashStamp, preReadBytes?: Buffer) => unknown {
const dir = path.dirname(dbPath)
// Per indexer, which drainOnce builds once per drain.
const embedPolicy = embedPolicyResolver()
return (absPath, sha, stamp, preReadBytes) => {
try {
// Register this file's project root as sweepable before any gate below can skip the file: a drain that finds everything fresh still proves the project has rows worth sweeping. See recordKnownRootThrottled.
recordKnownRootThrottled(absPath, dir, dbPath)
// Skip-eligibility must be checked UNCONDITIONALLY, before the parseUnchanged sha-gate: a file that becomes skip-eligible purely from a config change (same sha) would otherwise never reach indexFileSync's purge at all, leaving symbols/refs/files rows stale forever. Guard on ixCfg !== undefined for tests that mock loadConfig with a partial { worker: {...} } shape.
// One config read per file serves the skip check and the embed gates below: each loadConfig() opens and fingerprints config.toml and .token-goat.toml, and three of them per file were most of the file opens a drain of unchanged files made.
const ixCfg = loadConfig().indexing
if (ixCfg !== undefined && isParseSkipEligible(absPath, ixCfg)) {
// Purging stale rows is real work: unlike the `false` sha-gate skip below, this must count as "indexed" in processDirtyBatch's tally.
removeFileFromIndex(getDb(dbPath), absPath)
return true
}
// Same reason as the identical call in cli.ts's cmdIndex: files.embed_sha encodes the content but not the embedding stack that produced the vectors, so the gate below reads a file embedded by a previous model or inference runtime as fresh and returns "nothing to do". ensureEmbeddingProvenance owns that input and is the only thing that can re-open the decision. It has to run before getFileEntry below, not beside the gate that consults the result: the reset clears each affected file's embed_sha, and once `entry` holds a row that clearing can no longer be seen. Memoized per database per process, so this costs one Set lookup per drained file after the first. embeddingsEnabled/depsAvailable are hoisted here from beside the gate for the same ordering reason; see their comments there. The file's own project decides whether it is embedded, by the rule `token-goat index` uses too (see embedPolicyResolver), never the daemon's working directory, which is the temp directory (see runDetachedWorkerDaemon). The other indexing keys this function reads through loadConfig() with no root, `max_chunks_per_file` just below among them, are right from any root only because each is in PROJECT_LOCKED_KEYS, so no project file can set one; a key a project may set has to be read from `configRoot`.
const { configRoot, embeddingsEnabled } = embedPolicy(absPath)
const depsAvailable = embeddingsEnabled && embeddingsDepsAvailable(getDb(dbPath))
if (depsAvailable) {
ensureEmbeddingProvenance(getDb(dbPath))
// Same placement and the same reason as the call above, for the gates that reject a file rather than the stack that embedded it; see src/embed_backfill.ts. It throttles itself on two ledger rows, the release and the `indexing.max_chunks_per_file` it last swept under, so it sweeps once after a release or a change to that setting and otherwise costs a CREATE TABLE IF NOT EXISTS plus one primary-key SELECT per file.
pruneUnembeddableChunks(getDb(dbPath), ixCfg?.max_chunks_per_file ?? 0, deleteFileEmbeddings, { asset: assetEmbedSha, maxChunks: maxChunksEmbedSha })
}
const entry = getFileEntry(absPath, dbPath)
// Skip the syntactic reparse when content is byte-identical to what's already indexed (same fingerprint) so a touched-but-unchanged file is not needlessly reparsed. ...and not when the row's own spelling has gone stale under a case-only rename, which leaves the content identical and would otherwise pin the old spelling in place forever. See indexedPathSpellingIsStale. ...and not when the rows were written by a different version of the extraction logic. files.sha answers "has the content changed", which is only half the question: a parser change alters what gets extracted from content that never moved, and before parser_sha existed those files kept their old symbol set for as long as nobody edited them. Measured on a real index, 37 of 237 source files disagreed with what the same binary produced from scratch. An empty parserSha is a row written before the column existed and is correctly stale. The expected stamp is the one for the language the row itself records, so a fix to one adapter reparses that language's files and leaves every other language's rows alone. See parserFingerprintForLanguage.
const spellingStale = entry !== null && indexedPathSpellingIsStale(entry.filePath, absPath)
const parseUnchanged =
entry?.sha === sha &&
entry.parserSha === parserFingerprintForLanguage(entry.language) &&
!spellingStale
const indexAndRestamp = () => {
if (!parseUnchanged) {
indexFileSync(absPath, dbPath, preReadBytes, sha)
}
const embedEntry = parseUnchanged ? entry : getFileEntry(absPath, dbPath)
if (stamp !== undefined && embedEntry !== null) {
if (!restampVerifiedUnchanged(dbPath, [{ indexedPath: embedEntry.filePath, sha, stamp }])) {
throw new Error('Pre-hash freshness restamp failed')
}
}
return embedEntry
}
// Embedding freshness is gated INDEPENDENTLY of parse freshness (files.embed_sha, set only after indexFileEmbeddings actually commits -- see its doc comment in parser.ts). If a prior embedding attempt crashed or threw before stamping embed_sha, the parse-sha gate above would otherwise mask that forever: identical content would keep skipping the reparse AND skip re-embedding, leaving chunks permanently stale/missing. Re-check embed_sha against the current sha every time, even when the parse gate above skipped. While embeddings are currently disabled, indexFileEmbeddings stamps embed_sha with disabledEmbedSha(sha) instead of the bare sha (see its doc comment) so this gate can still hold and avoid re-entering indexFileEmbeddings on every drain of an unchanged file -- but a bare-sha match must never satisfy the gate while disabled, or a file that was only ever marker-stamped (never actually embedded) would look "unchanged" the instant embeddings are re-enabled, permanently skipping its real first embed. Optional chaining/fallback here is a defensive test-mock safety net, not a real production path: loadConfig() always returns a fully-populated, schema-validated config object in production. Several existing tests in this file mock loadConfig() with only a partial `{ worker: {...} }` shape (they exercise unrelated gates), so a bare `.indexing.embeddings_enabled` here would throw for those. Default to enabled (true), matching config.ts's own default, so this new gate check is a no-op for tests that never cared about embeddings. embeddingsEnabled and depsAvailable are computed above, before getFileEntry. depsAvailable lets isEmbedFresh distinguish a file that was skipped only because the optional embedding deps were absent (stamped an `unavailable:` marker) from one that was really embedded: the marker stays "fresh" while deps are still missing, but forces a re-embed the moment the model + sqlite-vec become usable. That independence covers files.parser_sha too: it answers which extractor wrote the symbol rows, never whether the stored vectors match this content, so a stale parser stamp must not drag this gate to false (it did, which re-embedded the whole index on every parser bump and made writeParseResult's embedShaToCarry dead for the waste it exists to prevent). spellingStale is the one parse-side condition that does belong here: a case-only rename leaves the content byte-identical, so isEmbedFresh would say fresh, but `chunks` rows are keyed by file_path and would keep the old spelling forever. Read the stamp the reparse above just wrote, not the one `entry` captured before it: writeParseResult clears the carried embed_sha when the reparse moved this file's embedding boundaries (see embeddingBoundariesMoved in parser.ts), and consulting the pre-parse row would defer that re-embed to some later drain -- which for a file nobody edits again is never.
// Keep the parsed rows and their pre-hash stamp atomic; IMMEDIATE is needed because the parser's nested write transaction becomes a savepoint.
const embedEntry =
stamp === undefined
? indexAndRestamp()
: getDb(dbPath).transaction(indexAndRestamp).immediate()
const embedUnchanged =
!spellingStale &&
isEmbedFresh(
embedEntry?.embedSha,
sha,
embeddingsEnabled,
depsAvailable,
// Same partial-config defensiveness as embeddingsEnabled above: 0 matches no stamped oversize marker (config floors this key at 1), so a mocked config re-examines the file rather than assuming it current. See isEmbedFresh.
ixCfg?.large_file_symbol_only_kb ?? 0,
// Same reasoning and same 0 fallback for the chunk-count marker.
ixCfg?.max_chunks_per_file ?? 0,
)
if (embedUnchanged) {
// Nothing left to embed. `false` tells processDirtyBatch the gate found nothing to do at all, so it is only truthful when the parse was skipped too: a parser-stamp bump reparses the file above while correctly leaving its embedding alone, and that is a real index rather than a no-op skip.
return parseUnchanged ? false : undefined
}
// Embeddings are fired and forgotten here, never awaited: the worker's drain loop is synchronous by design (drainOnce/processDirtyBatch must return instantly so the dirty queue clears promptly), and chunk/vector freshness can safely lag a beat behind symbol freshness since semantic search tolerates staleness in a way exact symbol lookups do not. indexFileEmbeddings already swallows its own errors internally, and embedFileSerialized ends its chain with a .catch so the promise returned here can never reject -- that backstop lives there, not on this line, and removing it would leave this un-awaited call able to take the daemon down with an unhandled rejection. Returning the promise (rather than voiding it) lets a caller that wants to - such as a test - await it explicitly instead of racing it. Routed through embedFileSerialized so two overlapping drains of the same rapidly re-edited file chain onto one another instead of racing -- see its doc comment for the stale-overwrite bug this closes. The model is not on this machine and its download is held (offline, or failed moments ago): an embed now would only fail again. The file keeps a NULL embed_sha, so runWorkerLoop's backlog sweep embeds it once embeddingsReady turns true.
if (modelDownloadHeld()) return parseUnchanged ? false : undefined
return embedFileSerialized(absPath, dbPath, sha, configRoot)
} catch (err) {
// One bad file must not abort the rest of the batch -- but a swallowed failure must not be silently indistinguishable from a successful index either. See the doc comment above.
logIndexFailure(dir, absPath, err)
return INDEX_FAILED
}
}
}
// Build the remove callback the drain loop uses by default: drop the index rows and embedding chunks for a path whose file has vanished from disk, reconciling deletions. A failure on one path is swallowed so a single bad delete never aborts the batch or crashes the drain loop.
function makeRemover(dbPath: string): (absPath: string) => void {
return (absPath) => {
removeFileFromIndex(getDb(dbPath), absPath)
}
}
/** Prune `p`'s rows through `remove` and clear its retry streak; on failure log it and requeue it within the same retry budget, since a failed prune leaves stale symbols just like a failed reindex. Returns whether the prune succeeded. */
function pruneDirtyPath(p: string, remove: (absPath: string) => void, dir: string, requeue: (dir: string, absPath: string) => void): boolean {
try {
remove(p)
clearRetryCount(path.join(dir, 'global.db'), p)
return true
} catch (err) {
appendWorkerErrorLog(dir, `${new Date().toISOString()} removeFileFromIndex failed for ${p}: ${extractErrorMessage(err)}\n`)
requeue(dir, p)
return false
}
}
/** Process one batch of dirty paths. For each path: skip if the file no longer exists or cannot be fingerprinted, otherwise re-index it. The default `index` callback parses the file and writes its rows into the global index DB; tests inject their own callback to observe the plumbing in isolation. Returns the number of paths actually (re)indexed -- a path whose callback returns `false` (the default indexer's sha-gate skip for byte-identical content) or `INDEX_FAILED` (the default indexer's sentinel for a swallowed, logged indexing failure -- see makeIndexer) is visited but not counted. Exported for unit tests so the drain logic can be exercised without a thread. Once the daemon starts retiring (see {@link drainShouldHandBack}), the paths not yet reached go back to the live queue, and this batch ends. */
export function processDirtyBatch(
paths: string[],
index: (
absPath: string,
sha: string,
stamp?: PreHashStamp,
preReadBytes?: Buffer,
) => unknown = makeIndexer(globalDbPath()),
remove: (absPath: string) => void = makeRemover(globalDbPath()),
dir: string = dataDir(),
requeue: (dir: string, absPath: string) => void = requeueDirtyPath,
handBack: (dir: string, absPath: string) => void = (handBackDir, absPath) => {
appendToDirtyQueue(handBackDir, absPath)
},
): number {
const blockedRoots = loadConfig().worker.blocked_roots
const ixCfg = loadConfig().indexing
// Memoizes findProject(dirname) for this batch only -- findProject re-runs a full ancestor-marker probe (9 markers x existsSync/lstat per level) plus isRepoContainer's readdirSync on every call, and a batch routinely contains many dirty paths under the same directory (or same project tree) that would otherwise repeat that walk once per path for a result (lastKnownProjectRoots) that is only consulted once per PRUNE_EVERY_N_DRAINS drains. Scoped to this function call (not module-level) since a project root learned from one batch's traffic can legitimately go stale by the next batch (e.g. after a project is deleted).
const projectRootCache = new Map<string, string | null>()
let indexed = 0
for (let i = 0; i < paths.length; i++) {
const p = paths[i]
if (!p) continue
if (drainShouldHandBack()) {
// No retry is charged, since nothing failed. Persisted before the caller clears the claimed file that named them, so a crash in between duplicates a path (its sha gate absorbs that) and never drops one.
const unreached = paths.slice(i).filter((q) => q.length > 0)
if (!appendToDirtyQueue(dir, ...unreached)) for (const q of unreached) handBack(dir, q)
break
}
writeDrainHeartbeat(dir)
// worker.blocked_roots (set via `token-goat project exclude`) excludes a path prefix from reindexing entirely -- skip before the existence/sha checks so a blocked path is never touched, not even pruned from the index if it happens to have been deleted underneath it.
if (isUnderBlockedRoot(p, blockedRoots)) continue
// A dirty path whose file is gone is a deletion to reconcile, not a no-op: prune its stale rows instead of skipping, otherwise `symbol Foo` resolves a deleted file forever. indexedFileIsGone, not fs.existsSync: a directory now standing at the path counts as gone (fingerprintFileWithBytes can never read it, so it was logged as a transient failure and abandoned after its retries, leaving the old file's symbols served), while existsSync answers false for a locked or permission-denied file exactly as it does for a missing one, so a file held open by an AV scanner or sitting behind a deny ACE would have had its symbols, references, sections and embedding chunks deleted while it was still on disk -- silently, and with no way back until something edited it again. That is the same class of transient failure the fingerprint branch immediately below already logs and requeues, so an unreadable file now falls through to it instead of being pruned.
if (indexedFileIsGone(p)) {
pruneDirtyPath(p, remove, dir, requeue)
continue
}
// Decided from the path and a stat before fingerprintFileWithBytes reads the whole file, as cmdIndex does: a log past large_file_skip_kb was otherwise loaded whole on every drain only to be skipped, and one over 2 GiB failed that read and was requeued as transient until its retries ran out, its stale rows never purged. Counted as indexed for the same reason as makeIndexer's identical branch, which still covers its direct callers.
if (ixCfg !== undefined && isParseSkipEligible(p, ixCfg)) {
recordKnownRootThrottled(p, dir, path.join(dir, 'global.db'))
if (pruneDirtyPath(p, remove, dir, requeue)) indexed += 1
continue
}
// Stat and clock before the hash, so the mtime recorded for a file whose sha matches its row is the one the hashed bytes carried and its index time is no later than the hash; makeIndexer's restamp relies on that order. A failed stat leaves it undefined and the restamp is skipped.
const stamp = readPreHashStamp(p)
const fingerprint = fingerprintFileWithBytes(p)
if (fingerprint === null) {
// The file exists but couldn't be read right now (lock/permission/race) -- see logTransientReadFailure's doc comment. Log and requeue instead of silently dropping it.
logTransientReadFailure(dir, p)
requeue(dir, p)
continue
}
// Opportunistically learn the active project root from this batch's traffic -- see lastKnownProjectRoots' doc comment for why this global, path-keyed queue has no other standing notion of "the current project" for drainOnce's periodic prune sweep to target.
try {
const dirname = path.dirname(p)
let root: string | null
if (projectRootCache.has(dirname)) {
root = projectRootCache.get(dirname) ?? null
} else {
const project = findProject(dirname)
root = project?.root ?? null
projectRootCache.set(dirname, root)
}
if (root) lastKnownProjectRoots.set(dir, root)
} catch {
// best-effort signal only -- never let project-root discovery abort a real index.
}
// `false` means the sha-gate skipped a no-op reindex; INDEX_FAILED means the default indexer's catch swallowed a genuine failure (logged separately -- see makeIndexer). Any other return value (including void/undefined from callers that don't bother returning anything) counts as indexed.
const result = index(p, fingerprint.sha, stamp, fingerprint.bytes)
if (result === INDEX_FAILED) {
// Requeue for the same reason the fingerprint branch above does, and it is the same class of failure: makeIndexer's catch covers a SQLITE_BUSY on the index write and a lock inside indexFileSync that the earlier fingerprint read happened to miss, both of which clear on their own. drainOnce has already claimed this batch by renaming the queue file aside and will delete it, so a path not requeued here is gone for good -- its rows then stay stale until something edits the file again, silently, which is worse than being visibly behind. The bounded budget in bumpAndCheckRetry is what keeps a genuinely unindexable file from being retried forever.
requeue(dir, p)
continue
}
// Clear the transient-retry streak only now, once the path is actually current, so a later failure starts from a full budget instead of resuming a much earlier streak (see requeueDirtyPath / bumpRetryCount). Deliberately after the index rather than after the read that preceded it: a successful read followed by a failed index has made no progress, and clearing there would hand a permanently failing file a fresh budget on every single cycle, so the requeue above would hammer it forever instead of giving up. `false` (the sha-gate found the content already indexed) is progress and belongs on this side of the branch; only INDEX_FAILED is not.
clearRetryCount(path.join(dir, 'global.db'), p)
if (result !== false) indexed += 1
}
return indexed
}
/** Synchronous 50ms sleep for the Windows rename-retry loop (mirrors Python's time.sleep(0.05)). Uses Atomics.wait on a SharedArrayBuffer to block without spinning; silently skips on platforms where SAB is unavailable. */
function sleepSyncMs(ms: number): void {
try {
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
} catch {
// SAB unavailable or other atomics error: skip sleep
}
}
/** Run one drain cycle for `dir`: atomically claim the dirty queue, process it. Atomically renames the live queue to a .draining file so that concurrent appendDirtyPath calls either land before the rename (and travel with it) or recreate a fresh queue after it (picked up on the next poll). This is the Python original's rename-to-claim pattern, preventing lost updates. Recovers from crashes by absorbing an abandoned .draining file at startup. On Windows, rename can fail with EPERM if the file is open for append; the loop retries 5 times with 50ms sleeps before deferring (returning 0). Each stage's claimed/recovered file is only cleared (rm'd or quarantined) AFTER its batch has been durably processed by {@link processDirtyBatch} -- never before. If the process dies partway through a batch (SIGTERM, a crash, or this daemon being killed as a duplicate after losing the worker_lifecycle.ts::claimWorkerPidFile startup race), the .draining file is still on disk for the next startup's crash recovery to pick back up, instead of having already been deleted while the paths it named were never indexed. Returns the number of paths processed; the count no longer includes reconcile sweeps, which {@link runWorkerLoop} awaits after each cycle (see reconcile_requests.ts::drainReconcileRequestsAsync) because they run git without blocking. When no `index` callback is injected, files are indexed into `dir`'s `global.db` (the real shipping path); in production `dir` is the data dir, so this is {@link globalDbPath}. */
export function drainOnce(
dir: string,
index?: (absPath: string, sha: string) => unknown,
remove?: (absPath: string) => void,
): number {
const queuePath = dirtyQueuePathFor(dir)
const draining = `${queuePath}.draining`
const dbPath = path.join(dir, 'global.db')
const indexFn = index ?? makeIndexer(dbPath)
const removeFn = remove ?? makeRemover(dbPath)
let processed = 0
// A transient read failure in stage (a) (recovering an abandoned .draining file) must not be requeued straight onto the live dirty.txt: stage (b) below claims that same live queue microseconds later, in this SAME drainOnce call, and would immediately reprocess the path while the lock/condition that caused the original failure has almost certainly not cleared yet -- double-bumping its retry count once per cycle instead of once, roughly halving the effective MAX_TRANSIENT_RETRIES budget. Collect this cycle's requeues here and only write them to the live queue once BOTH stages have finished claiming/processing their batches, so a requeued path is guaranteed to wait for the NEXT drainOnce cycle. bumpAndCheckRetry (which increments the retry counter and enforces MAX_TRANSIENT_RETRIES) still runs immediately, at the point of failure -- only the disk append is deferred.
const deferredRequeues: string[] = []
// Each deferred requeue is also written at once to a sidecar named like a stage (b) fallback claim, because the queue files that named these paths are deleted before the live append below: a kill, or a throw out of stage (b), in between would otherwise lose them. This cycle's stage (a) listed its files before the sidecar existed, so only a later cycle recovers it, and the deferral holds.
const requeueSidecar = `${draining}.alt-requeue-${process.pid}-${Date.now()}`
const deferAppend = (absPath: string): void => {
deferredRequeues.push(absPath)
try {
appendDirtyQueueFile(requeueSidecar, [absPath])
} catch {
// The in-memory copy still reaches the live queue at the end of this cycle; only a kill before then loses it.
}
}
const requeueFn = (requeueDir: string, absPath: string): void => {
if (!bumpAndCheckRetry(requeueDir, absPath)) return
deferAppend(absPath)
}
// Unreached paths of a retiring drain: deferred the same way, but never charged a retry, since nothing failed.
const handBackFn = (_handBackDir: string, absPath: string): void => deferAppend(absPath)
// (a) Crash recovery: absorb any `.draining` (and `.draining.alt-*` fallback) files abandoned by a previous crashed or stuck drain. Multiple files can accumulate when a Windows sharing violation keeps the primary `.draining` name locked across cycles (see stage (b)'s fallback-claim comment) -- recover every one of them, not just the first, so a single stuck file can never starve the rest of the queue from ever draining.
const liveDrainingFiles = listDrainingFiles(queuePath)
// Drop snapshots for draining files that no longer exist. An entry is only added when both cleanup attempts failed, and is otherwise removed when its file is cleared -- but a file deleted or quarantined by anything else leaves its entry behind forever, so the map grows without bound across cycles and each value holds a whole queue file's contents. A stale entry is also wrong, not merely large: if a later cycle recreates the same draining name with byte-identical content, the leftover snapshot makes the guard below skip a batch that was never processed.
const liveDrainingSet = new Set(liveDrainingFiles)
for (const key of unclearedDrainingSnapshots.keys()) {
if (!liveDrainingSet.has(key) && key.startsWith(`${queuePath}.draining`)) unclearedDrainingSnapshots.delete(key)
}
for (const drainingFile of liveDrainingFiles) {
let drainingContent: string | null = null
// A read failure here is not proof the file is corrupt: on Windows an antivirus scan or another process holding the file open fails the read while a rename still succeeds, and quarantining on the first failure loses every path the file named, permanently -- listDrainingFiles excludes `.corrupt-` names from recovery and cleanupWorkerStateFiles deletes them after 30 days. Retry on the same schedule stage (b) uses for its claim rename before concluding the file is genuinely unreadable.
for (let attempt = 0; attempt < DRAINING_READ_ATTEMPTS; attempt++) {
try {
drainingContent = fs.readFileSync(drainingFile, 'utf8')
break
} catch {
if (attempt < DRAINING_READ_ATTEMPTS - 1) sleepSyncMs(DRAINING_READ_RETRY_DELAY_MS)
}
}
if (drainingContent === null) {
// Still unreadable after every retry: quarantine so it cannot collide with a future claim, then skip it this cycle.
try {
fs.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`)
// The file is gone under its old name, so any snapshot keyed to it now describes a path that no longer exists.
unclearedDrainingSnapshots.delete(drainingFile)
} catch {
// best effort
}
continue
}
// Only process this content if it was not already folded into a batch on a prior cycle (see unclearedDrainingSnapshots below). Without this guard, a draining file that outlives both cleanup attempts (e.g. a persistent Windows sharing violation) would be re-read and its paths reprocessed on every cycle.
if (unclearedDrainingSnapshots.get(drainingFile) !== drainingSnapshotStamp(drainingFile, drainingContent)) {
processed += processDirtyBatch(parseDirtyQueueLines(drainingContent), indexFn, removeFn, dir, requeueFn, handBackFn)
}
// Only clear the recovered file now that its batch has been durably processed (or recognized above as already processed) -- never before -- so a crash partway through processDirtyBatch leaves the file in place for the next startup to recover, instead of deleting it up front and losing every path it named.
try {
fs.rmSync(drainingFile, { force: true })
unclearedDrainingSnapshots.delete(drainingFile)
} catch {
try {
fs.renameSync(drainingFile, `${drainingFile}.corrupt-${Date.now()}`)
unclearedDrainingSnapshots.delete(drainingFile)
} catch {
// Both cleanup attempts failed and the file is still stuck: remember exactly what we already folded into this cycle's batch so the next cycle can retry cleanup without reprocessing the same paths again.
unclearedDrainingSnapshots.set(drainingFile, drainingSnapshotStamp(drainingFile, drainingContent))
}
}
}
// (b) Atomically claim the live queue. A concurrent appendDirtyPath either landed before the rename (its line travels in .draining) or recreates a fresh dirty.txt after it (next cycle) — it can never be deleted unindexed. On Windows a concurrent open-for-append can make rename fail with EPERM/ EBUSY/EEXIST; retry a few times, then defer (return 0 = retry next poll).
// A leaving daemon claims nothing new: whatever is queued stays in the live queue for the next daemon.
if (!drainShouldHandBack() && fs.existsSync(queuePath)) {
// If the primary `.draining` name is still occupied by a file the loop above could not clean up (both rmSync and rename-to-corrupt failed -- typically a Windows sharing violation from something else holding it open), claiming into that same name would collide forever and starve the live queue indefinitely. Claim into a distinct `.alt-<ts>` name instead; listDrainingFiles recovers it on a future cycle.
const claimTarget = fs.existsSync(draining) ? `${draining}.alt-${Date.now()}` : draining
let claimed = false
for (let attempt = 0; attempt < 5; attempt++) {
try {
fs.renameSync(queuePath, claimTarget)
claimed = true
break
} catch {
sleepSyncMs(50)
}
}
if (claimed) {
let claimedContent = ''
let readOk = false
try {
claimedContent = fs.readFileSync(claimTarget, 'utf8')
readOk = true
} catch {
// read failure is fail-soft: leave the claimed file in place; the next cycle's stage (a) crash recovery will pick it up.
}
if (readOk) {
// Deliberately NOT wrapped in the try above: a throw from processDirtyBatch (e.g. the process crashing mid-batch) must propagate to the caller, not be swallowed as a "read failure", so the cleanup below never runs and the claimed file survives.
processed += processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn, handBackFn)
// Re-read the claimed file ONE more time, right before deleting it, to catch a path appended by a writer that raced the claim-rename above. The rename-to-claim pattern assumes no other writer can touch the live queue's underlying file once it has been renamed to the claim target -- but that isn't guaranteed on Windows (FILE_SHARE_DELETE lets a concurrent open-for-append follow the same underlying file object across the rename) or on POSIX (a narrow rename/open race exists there too). A dirty path appended during that window would otherwise be silently deleted along with the rest of this cycle's already-processed content and never reindexed. Forward anything found beyond what was already processed back into the live queue so the next drain cycle picks it up.
try {
const recheck = fs.readFileSync(claimTarget, 'utf8')
if (recheck !== claimedContent) {
const extra = recheck.startsWith(claimedContent) ? recheck.slice(claimedContent.length) : recheck
for (const p of parseDirtyQueueLines(extra)) appendToDirtyQueue(dir, p)
}
} catch {
// best-effort recheck -- if the claimed file vanished or became unreadable between the read above and now, there is nothing more we can safely recover here.
}
// Only clear the claimed file now that its batch has been durably processed -- never before -- so a crash partway through processDirtyBatch leaves it in place for stage (a) to recover on the next startup instead of losing the paths it named.
try {
fs.rmSync(claimTarget, { force: true })
} catch {
try {
fs.renameSync(claimTarget, `${claimTarget}.corrupt-${Date.now()}`)
} catch {
// Both cleanup attempts failed: record it the same way stage (a) does, so a leftover file here is recognized as already-processed (and not silently reprocessed) by stage (a)'s crash recovery on the next cycle.
unclearedDrainingSnapshots.set(claimTarget, drainingSnapshotStamp(claimTarget, claimedContent))
}
}
}
}
// If the claim-rename never succeeded after 5 retries, the live queue is left untouched and will be retried on the next poll cycle.
}
// Now that both stages above have finished claiming/processing their batches for this cycle, it is safe to actually append this cycle's transient-failure requeues to the live queue -- see deferredRequeues' doc comment above for why this must not happen sooner. It runs before the prune sweep so a slow sweep does not hold them in memory, and the sidecar goes only once they are in the live queue.
if (deferredRequeues.length > 0 && appendToDirtyQueue(dir, ...deferredRequeues)) {
try {
fs.rmSync(requeueSidecar, { force: true })
} catch {
// A sidecar left behind is recovered by the next cycle's stage (a), which costs one duplicate unchanged-sha skip per path.
}
}
// (c) Opportunistic prune sweep for renamed/deleted files that never enqueued via the Edit hook path (git mv, git checkout, git clean). Runs after all normal dirty-queue work above, on a low cadence (see PRUNE_EVERY_N_DRAINS' doc comment), so it never delays draining the queue itself and a slow sweep on a huge repo only pushes out the NEXT sweep's schedule, not this cycle's already-completed dirty-queue work.
const cycle = (drainCycleCounts.get(dir) ?? 0) + 1
drainCycleCounts.set(dir, cycle)
if (cycle % PRUNE_EVERY_N_DRAINS === 0) {
const root = lastKnownProjectRoots.get(dir)
if (root !== undefined) {
try {
pruneDeletedFiles(root, dbPath)
} catch {
// Best-effort housekeeping; a prune failure must not kill the drain loop.
}
}
}
// Touch the heartbeat marker last even when the queue was empty, and keep it fresh during long batches through processDirtyBatch.
writeDrainHeartbeat(dir, true)
return processed
}
/** The drain loop itself. Sleeps between cycles via a Promise + setTimeout so the thread/process stays responsive to termination. `shouldStop` lets callers (and tests) break the loop deterministically; in the worker-thread case it is wired to a message from the parent. */
// How often the worker loop auto-prunes dead file rows across every known project root (see sweepKnownRoots in index_prune.ts). Deliberately much longer than SNAPSHOT_CLEANUP_INTERVAL_MS -- a full existence-check pass over every indexed file under every known root is heavier than the snapshot sweep, and dead-row accumulation is a slow-moving problem that doesn't need a tight cadence to stay bounded.
const KNOWN_ROOTS_SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000
// How often the worker loop performs automatic off-peak idle vacuuming / space reclamation on global.db. Runs only when the worker has been continuously idle (no dirty items) for at least IDLE_VACUUM_COOLDOWN_MS.
const IDLE_VACUUM_INTERVAL_MS = 6 * 60 * 60 * 1000
const IDLE_VACUUM_COOLDOWN_MS = 5 * 60 * 1000
// Smallest share of the database file that must be free pages before the idle pass vacuums it. VACUUM rewrites every page, and in WAL mode it writes them through the log, so on a 5 GB database with nothing to give back it costs a 5 GB write and a 5 GB -wal for no gain; the idle pass runs every six hours, so it must not be that expensive when there is nothing to reclaim.
const IDLE_VACUUM_MIN_FREE_SHARE = 0.1
/** One idle reclaim pass over the database at `dbPath`: vacuum it when at least {@link IDLE_VACUUM_MIN_FREE_SHARE} of the file is free pages, otherwise only fold and truncate its -wal (a checkpoint writes the live frames once and rewrites nothing else). True when the pass is settled for this interval (vacuumed, or nothing worth reclaiming); false when there is no database yet, the vacuum could not get its lock, or a reader refused the truncating checkpoint (after a vacuum or alone), so the next cycle tries again. That retry is a checkpoint that does not wait for the reader and, while the reader still holds the log, copies no frames (those written after the reader's snapshot stay in the log until it leaves); the vacuumed file has no free pages left, so it is never a second rewrite. */
export function idleReclaim(dbPath: string): boolean {
if (!fs.existsSync(dbPath)) return false
const db = getDb(dbPath)
const free = Number(db.pragma('freelist_count', { simple: true }))
const total = Number(db.pragma('page_count', { simple: true }))
if (!(total > 0) || free / total < IDLE_VACUUM_MIN_FREE_SHARE) {
// A reader that refuses the truncation leaves the log full-size, so the pass is not settled and the next cycle asks again; the checkpoint does not wait, so asking every cycle costs a lock attempt and no page copying while the reader stays, and stalls nothing.
return !walCheckpointBusyNoWait(db)
}
// Neither of the pass's own checkpoints waits for a reader either: while one stays, a wait would hold the whole worker for the connection's busy_timeout on every poll.
const reclaimed = reclaimIndex(dbPath, { rebuild: false, waitForReaders: false })
// A vacuum rewrites every page through the log, so a refused truncation after it leaves a log as large as the database. The retry cannot vacuum again (no free pages remain) and is the non-waiting checkpoint above.
return !reclaimed.vacuumDeferred && !reclaimed.checkpointBusy
}
/** The run of idle reclaim polls that were not settled, as the loop sees it: how many so far since the last one that settled. */
export interface IdleReclaimRefusals {
polls: number
}
/** Write the worker error log's two lines for a run of refused idle reclaims: one when the first poll of the run is refused (the database and why), and one when a later poll settles (how many were refused). Polls in between write nothing, so a reader that stays for hours costs two lines. `error` is set when the pass threw instead of answering false. */
export function logIdleReclaimPoll(dir: string, dbPath: string, run: IdleReclaimRefusals, poll: { settled: boolean; error?: unknown }): void {
if (!poll.settled) {
run.polls++
if (run.polls > 1) return
const why = poll.error === undefined ? 'a reader is holding the -wal or the database is locked' : extractErrorMessage(poll.error)
appendWorkerErrorLog(dir, `${new Date().toISOString()} idle reclaim of ${dbPath} was refused (${why}); asking again each idle cycle without waiting\n`)
return
}
if (run.polls === 0) return
appendWorkerErrorLog(dir, `${new Date().toISOString()} idle reclaim of ${dbPath} settled and the -wal was truncated after ${run.polls} refused ${run.polls === 1 ? 'poll' : 'polls'}\n`)
run.polls = 0
}
// How many files owed an embed the worker puts back on its queue at a time. The next batch waits until every embed in flight has settled, so a backlog of any size is worked through at the pace the embedding slots allow and never ahead of an edit that arrives meanwhile.
const EMBED_BACKLOG_BATCH = 50
// How many `files` rows one idle cycle reads looking for that batch. Reading a row is an index walk with no file I/O, so a large index with little owed is crossed in a few hundred cheap cycles.
const EMBED_BACKLOG_SCAN = 1000
/** Put up to {@link EMBED_BACKLOG_BATCH} files whose embedding is not current back on the dirty queue, reading from after `cursor` in path order, and return where the next cycle resumes, or null once the walk is done. Embedding is queued in this process's memory while the drain moves on (see makeIndexer), so a worker that stops with a backlog loses it: a SIGTERM waits five seconds for embeds in flight and no longer, and on Windows, where `worker stop` terminates the process outright, nothing waits at all. Every file it drops keeps a NULL `embed_sha`, which no later drain revisits, because a drain only sees files something touches again. That left 25,840 of one project's 27,051 files out of 'semantic' after a single large reindex. A stamp that the configuration has since overtaken is the same backlog by another route: turning embeddings back on leaves every `disabled:` stamp stale, and raising indexing.large_file_symbol_only_kb leaves every `oversize:` stamp taken under the old value stale, and neither is revisited either. So the test is makeIndexer's own freshness gate, {@link isEmbedFresh} under the current configuration, and each file it rejects takes makeIndexer's ordinary path, where an unchanged file skips the reparse and goes straight to the embed. The worker loop does not call it while embeddings are off or their dependencies are missing: every file would only be stamped `disabled:` or `unavailable:`, which reads each one for no result, and the stamp it replaces already answers `semantic` the same way. Switching them back on starts the walk again from the top, because every file indexed in between owes an embed wherever the walk had got to. The cursor only moves forward, so a file whose embed fails every time is tried once per worker start rather than on every idle cycle. */
function requeueStaleEmbeddings(dir: string, cursor: string): string | null {
const indexing = loadConfig().indexing
const db = getDb(path.join(dir, 'global.db'))
const symbolOnlyKb = indexing?.large_file_symbol_only_kb ?? 0
const maxChunks = indexing?.max_chunks_per_file ?? 0
const rows = db.prepare('SELECT path, sha, embed_sha FROM files WHERE path > ? ORDER BY path LIMIT ?').all(cursor, EMBED_BACKLOG_SCAN) as Array<{ path: string; sha: string; embed_sha: string | null }>
const owed: string[] = []
// One per cycle, so a project with every file stamped `disabled:` costs one config read per cycle rather than one per row.
const embedPolicy = embedPolicyResolver()
let resumeAfter: string | null = null
for (const row of rows) {
resumeAfter = row.path
if (isUnderSystemTemp(row.path) || isEmbedFresh(row.embed_sha ?? undefined, row.sha, true, true, symbolOnlyKb, maxChunks)) continue
// A project whose `.token-goat.toml` turns embeddings off owes none, for the reason this sweep does not run while they are off everywhere: queuing its files would only stamp them `disabled:`, which answers `semantic` as the stamp it replaces does. Reading the setting from the daemon's working directory instead re-embedded every file `token-goat index` had just stamped disabled in that project.
if (!embedPolicy(row.path).embeddingsEnabled) continue
// The drain prunes a queued path that is not on disk, and a file on a drive that is only unmounted is not on disk either; deciding that a root is gone is sweepKnownRoots' job, behind its grace period.
if (fileIsAbsent(row.path)) continue
owed.push(row.path)
if (owed.length === EMBED_BACKLOG_BATCH) break
}
// A batch that never reached the queue is read again next cycle rather than stepped past.
if (!appendToDirtyQueue(dir, ...owed)) return cursor
return owed.length === EMBED_BACKLOG_BATCH || rows.length === EMBED_BACKLOG_SCAN ? resumeAfter : null
}
/** Whether an embed could be written now: the index exists, the configuration has not turned embeddings off, their dependencies load, and the model is on disk or may be downloaded. Both probes behind the dependency check are cached, so asking on every idle cycle costs two small config reads and a stat of the model files. */
function embeddingsReady(dir: string): boolean {
const dbPath = path.join(dir, 'global.db')
return fs.existsSync(dbPath) && loadConfig().indexing?.embeddings_enabled !== false && embeddingsDepsAvailable(getDb(dbPath)) && !modelDownloadHeld()
}
/** How long a daemon runs with an empty queue and nothing left to embed before it exits on its own; ensureWorkerAlive starts a new one on the next enqueue, so an unused data dir costs nothing but its hourly housekeeping. */
const DEFAULT_IDLE_EXIT_MS = 30 * 60 * 1000
/** The idle period after which a daemon exits: a positive-integer `TG_WORKER_IDLE_EXIT_MS`, else 30 minutes. Anything else in the variable (empty, non-numeric, zero, negative) is ignored, as `TG_WORKER_POLL_MS` is. */
export function resolveIdleExitMs(): number {
const parsed = parseInt(process.env['TG_WORKER_IDLE_EXIT_MS'] ?? '', 10)
return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_IDLE_EXIT_MS
}
/** True only when the pid file is definitely not there (ENOENT), as opposed to empty, mid-replacement or unreadable, which {@link readPidFile} also reports as null. A daemon retires on the first, never on the others. */
export function pidFileIsAbsent(dir: string): boolean {
try {
fs.statSync(workerPidPath(dir))
return false
} catch (e) {
return (e as NodeJS.ErrnoException).code === 'ENOENT'
}
}
/** When a daemon retires on its own, beyond a stop request: `pidFileGraceMs` is how long its pid file may be missing, or may not yet name it, before it leaves (default {@link WORKER_STARTUP_GRACE_MS}); `idleExitMs` is how long it may run with an empty queue and nothing to embed (default {@link resolveIdleExitMs}). */
/** The error-log line for a known-roots sweep that skipped roots whose dead-row ratio looked like a mount outage; the noun agrees with the count. */
export function flaggedRootsLogLine(flaggedRoots: readonly string[], isoTime: string): string {
return `${isoTime} sweepKnownRoots flagged ${countNoun(flaggedRoots.length, 'root')} for anomalously large dead-row ratio (skipped, not pruned): ${flaggedRoots.join(', ')}\n`
}
export interface WorkerRetirePolicy {
pidFileGraceMs?: number
idleExitMs?: number
}
export async function runWorkerLoop(
dir: string,
pollIntervalMs: number,
shouldStop: () => boolean = () => false,
retire?: WorkerRetirePolicy,
): Promise<void> {
const pidFileGraceMs = retire?.pidFileGraceMs ?? WORKER_STARTUP_GRACE_MS
const idleExitMs = retire?.idleExitMs ?? resolveIdleExitMs()
// The drain indexes files, and the regex language adapters it needs are behind a dynamic import (see loadRegexExtractors): load them before the first cycle, since drainOnce and the index callback under it are synchronous.
await loadRegexExtractors()
// Local to this loop invocation (not module-level) so each call starts its own fresh throttle window instead of sharing state across unrelated runWorkerLoop calls (e.g. across tests in the same process).
let lastSnapshotCleanupMs = 0
let lastKnownRootsSweepMs = 0
let lastIdleVacuumMs = 0
const idleReclaimRefusals: IdleReclaimRefusals = { polls: 0 }
let lastActiveMs = Date.now()
let embedBacklogCursor: string | null = ''
let embeddingsWereReady = true
let disabledStampsSeen = disabledStampCount()
// Flips true the first time we see the pid file naming our own pid (the parent claims it shortly after spawning us, so early polls may see it empty). Once set, losing ownership means another daemon took over -- see the self-terminate check below.
let ownedPidFile = false
// When this loop started, for the grace a daemon gets to see its own pid before a pid file naming someone else retires it.
const startedMs = Date.now()
// Since when the pid file has been definitely absent (ENOENT), or null while it is there.
let pidFileAbsentSinceMs: number | null = null
// The last time a cycle had anything to do: a drained batch, an embed in flight, or a backlog walk still going. Idle exit counts from here.
let lastBusyMs = Date.now()
// Ends each cycle's sleep early when a hook or command appends to the queue, so an edit is indexed within milliseconds rather than up to a poll interval later; the interval stays the fallback wherever the watch cannot run. See createQueueWaker. Every step of a cycle catches its own errors, so the loop leaves only through its exits below, each of which reaches the close.
const waker = createQueueWaker(dir)
// The retirement decision from the pid file, shared by the top of each cycle and by the work inside a cycle (see retiringNow). It takes the first-ownership step and the absence/grace bookkeeping, so it must be asked on every real check, not only once per cycle.
const pidFileSaysRetire = (): boolean => {
const pidOwner = readPidFile(dir)
if (pidOwner === process.pid) {
// First time this daemon owns the slot: end the daemon the claim displaced, off the hook path that spawned us.
if (!ownedPidFile) retireDisplacedWorker(dir)
ownedPidFile = true
} else if (ownedPidFile && pidOwner !== null) return true
const nowMs = Date.now()
if (pidFileIsAbsent(dir)) {
pidFileAbsentSinceMs ??= nowMs
if (nowMs - pidFileAbsentSinceMs >= pidFileGraceMs) return true
} else {
pidFileAbsentSinceMs = null
if (!ownedPidFile && pidOwner !== null && pidOwner !== process.pid && nowMs - startedMs >= pidFileGraceMs) return true
}
return false
}
// Asked from inside a drain, which can run far longer than the grace. Probed at most once per RETIRE_PROBE_INTERVAL_MS, and once it has said retire it stays so.
let retireLatched = false
let lastRetireProbeMs = 0
const retiringNow = (): boolean => {
if (retireLatched) return true
const nowMs = Date.now()
if (nowMs - lastRetireProbeMs < RETIRE_PROBE_INTERVAL_MS) return false
lastRetireProbeMs = nowMs
retireLatched = pidFileSaysRetire()
return retireLatched
}
retireCheckForDrain = retiringNow
while (!shouldStop()) {
// Self-terminate once this daemon's own data dir no longer exists: a caller that spawned a detached daemon against an ephemeral/scratch data dir (e.g. `token-goat index --walk` in a temp directory during dogfooding or a test run) and then deletes that directory without an explicit `worker stop` leaves the daemon with nothing left to poll -- `dirty.txt`/the pid file/global.db are all gone, so every subsequent drainOnce/cleanup call below is pure wasted work against a directory that will never come back. Without this check the daemon runs forever (confirmed in practice: 524 stray `--worker-daemon` processes accumulated over two weeks of dogfooding/test scratch-dir cleanup with no corresponding `worker stop`).
if (!fs.existsSync(dir)) break
// Self-terminate once another daemon has taken over this data dir's pid file. claimWorkerPidFile's reclaim path takes the pid file from a daemon whose heartbeat lease lapsed without signalling it (by then the pid may name an unrelated process), so this check is what retires a superseded daemon that is still running: without it that daemon keeps draining the same queue forever, because the existsSync check above never fires while the shared data dir still exists -- exactly how several stray daemons accumulate over successive sessions. Once we have observed the pid file naming our own pid, a later poll that finds it naming a *different* pid means we lost ownership, so exit and leave the current owner as the sole drainer. We only act on a concrete different pid, never on readPidFile returning null: null also covers a transient read failure or a mid-write empty file (pid-file replacement is not atomic to a concurrent reader), and treating that as lost ownership would let a single filesystem hiccup terminate the sole legitimate daemon -- a self-inflicted version of the very leak this check exists to stop. A genuinely removed pid file leaves us running until either fs.existsSync(dir) fires or a real successor writes its own pid, which is the safe direction.
if (pidFileSaysRetire()) break
// The two cases the takeover check above cannot see, both of which strand a daemon that nothing can then find or stop. A pid file that is gone for longer than the grace: `stopWorker` and a `rm -rf` of the data dir both remove it, and on Windows the second cannot remove the rest (this daemon holds global.db and a watch on queue/ open), so the directory-gone exit never fires; a single missing read is still ignored, since a successor claiming the slot removes and recreates the file. And a pid file naming another process after a whole grace in which it never named us: the daemon it was handed to lost the slot before it ever owned it.
let processed = 0
try {
processed = drainOnce(dir)
} catch {
// A bad batch must not kill the daemon; skip and retry next cycle.
}
if (retiringNow()) break
// Projects a hook asked to have swept because it could not list the files a branch switch changed (see reconcile_requests.ts). After the dirty queue so the sweep's own enqueues wait for the next cycle, as the deferred requeues in drainOnce do. Awaited here rather than run inside drainOnce, which is synchronous: the sweep's git must not block the event loop. Counts as work so an idle daemon does not retire in the middle of a catch-up.
try {
processed += await drainReconcileRequestsAsync(dir)
} catch {
// The sweep never throws; a request it left claimed is swept again next cycle.
}
if (retiringNow()) break
const foregroundBusy = hasRecentForegroundActivity()
if (processed > 0) {
lastActiveMs = Date.now()
lastBusyMs = lastActiveMs
} else if (inFlightEmbeddings.size === 0 && !foregroundBusy) {
// Only with the queue drained, every embed settled, and no active foreground hook turn, so the backlog never competes with an edit or interactive CLI turn. See requeueStaleEmbeddings.
try {
const ready = embeddingsReady(dir)
const stamps = disabledStampCount()
// Embeddings back on, or their dependencies back: every file indexed in between owes an embed wherever the walk had got to, a walk that already finished included, so it starts again from the top. Only idle cycles look, so a switch off and back on while the queue kept the worker busy shows up only as the `disabled:` markers its drains stamped meanwhile.
if (ready && (!embeddingsWereReady || stamps !== disabledStampsSeen)) embedBacklogCursor = ''
embeddingsWereReady = ready
if (ready) disabledStampsSeen = stamps
if (ready && embedBacklogCursor !== null) {
embedBacklogCursor = requeueStaleEmbeddings(dir, embedBacklogCursor)
// A walk that has not reached its end is work the next cycle continues.
if (embedBacklogCursor !== null) lastBusyMs = Date.now()
}
} catch {
// An unreadable index this cycle: the cursor stays put and the next idle cycle tries again.
}
}
// Sweep stale session-snapshot directories on the same periodic loop as the dirty-queue drain above, so accumulating session_snapshots/<sessionId>/ dirs get cleaned up on a schedule instead of growing unbounded for the life of the daemon. Deferred while a foreground CLI turn is active so synchronous directory scans do not compete with interactive tool calls.
if (!foregroundBusy && Date.now() - lastSnapshotCleanupMs >= SNAPSHOT_CLEANUP_INTERVAL_MS) {
try {
cleanup_stale()
} catch {
// Best-effort housekeeping; a cleanup failure must not kill the daemon either.
}
// Rotate/prune the worker's own accumulating state files (worker-errors.log, .corrupt-* quarantine files) on the same periodic cadence -- see cleanupWorkerStateFiles' doc comment.
try {
cleanupWorkerStateFiles(dir)
} catch {
// Best-effort housekeeping; a cleanup failure must not kill the daemon either.
}
// Age out the id-keyed disk caches on the same cadence. storeBlob prunes the subdir it writes, but session state is written outside that funnel and so had no reaper at all -- see sweepCacheRoots' doc comment. `dataDir()` is passed as a second root because older versions kept these caches there.
try {
sweepCacheRoots([dataDir()])
} catch {
// Best-effort housekeeping; a cleanup failure must not kill the daemon either.
}
// The shrunk image copies hosts read from the OS temp dir were otherwise swept only by the next copy a process writes, so the last ones of a session stayed indefinitely. The sweep keeps anything inside its one-hour delivery window.
sweepStaleShrinkCopies()
lastSnapshotCleanupMs = Date.now()
}
// Auto-prune dead file rows across every known project root -- see sweepKnownRoots' docstring for the safety guarantees (grace period before treating an unreachable root as gone, anomaly-ratio guard against a mount-point outage under a live root). This is what keeps the shared global.db from silently accumulating dead rows indefinitely the way it used to, when pruning only ever ran via a manually-invoked `token-goat index`.
if (!foregroundBusy && Date.now() - lastKnownRootsSweepMs >= KNOWN_ROOTS_SWEEP_INTERVAL_MS) {
try {
const result = sweepKnownRoots(path.join(dir, 'global.db'))
if (result.flaggedRoots.length > 0) {
appendWorkerErrorLog(dir, flaggedRootsLogLine(result.flaggedRoots, new Date().toISOString()))
}
} catch {
// Best-effort housekeeping; a sweep failure must not kill the daemon either.
}
lastKnownRootsSweepMs = Date.now()
}
// Perform off-peak idle space reclamation / vacuum on global.db when the daemon has been continuously idle (no dirty items) for at least IDLE_VACUUM_COOLDOWN_MS and at least IDLE_VACUUM_INTERVAL_MS has elapsed since the last vacuum.
const now = Date.now()
if (!foregroundBusy && now - lastIdleVacuumMs >= IDLE_VACUUM_INTERVAL_MS && now - lastActiveMs >= IDLE_VACUUM_COOLDOWN_MS) {
const reclaimTarget = path.join(dir, 'global.db')
// No database yet is nothing to reclaim, not a refusal: it neither counts nor logs, and the mark stays put so the next cycle looks again. idleReclaim keeps its own false for a missing file, which tests/wal_size_bounded.test.ts pins.
if (fs.existsSync(reclaimTarget)) {
try {
const settled = idleReclaim(reclaimTarget)
if (settled) lastIdleVacuumMs = now
logIdleReclaimPoll(dir, reclaimTarget, idleReclaimRefusals, { settled })
} catch (err) {
// Best-effort non-blocking maintenance; a pass that threw counts as refused so the log still says the reclaim is not getting done.
logIdleReclaimPoll(dir, reclaimTarget, idleReclaimRefusals, { settled: false, error: err })
}
}
}
if (shouldStop()) break
if (inFlightEmbeddings.size > 0) lastBusyMs = Date.now()
// Nothing to drain, nothing to embed, for the whole idle period: leave. The exit handler removes the pid file, and the next enqueue's ensureWorkerAlive starts a fresh daemon, so an abandoned data dir stops costing a process while a live one loses nothing but its housekeeping.
if (Date.now() - lastBusyMs >= idleExitMs) break
await waker.sleep(pollIntervalMs)
}
retireCheckForDrain = null
waker.close()
}
/** Run the detached daemon's drain loop in the current (main-thread) process. Reads its poll interval and data dir from the `TG_WORKER_POLL_MS` / `TG_WORKER_DATA_DIR` env vars set by worker_lifecycle.ts::startDetachedWorker on the child it spawns, registers a SIGTERM handler for a clean exit, and starts {@link runWorkerLoop} without awaiting it -- the loop's own setTimeout chain keeps the event loop (and therefore the process) alive indefinitely. This must be called explicitly by the CLI entrypoint (`cli.ts`'s `run()`) when `--worker-daemon` is present in argv, BEFORE commander ever sees argv: `--worker-daemon` is not a registered commander option or command anywhere in `buildProgram`, so letting commander parse first makes it reject the flag as unknown and the freshly-spawned daemon child exits immediately. This is the sole trigger point for the daemon loop in the shipped CLI -- nothing else should call it, since {@link runWorkerLoop} would then be running twice against the same dirty queue. Registers a `process.on('exit', ...)` handler that clears this daemon's own pid file so any exit path other than a clean worker_lifecycle.ts::stopWorker call (the SIGTERM handler above, an uncaught exception, or the process simply crashing) doesn't leave a stale pid file behind forever. The handler only removes the file when it still names this exact process -- never unconditionally -- so a daemon that lost the worker_lifecycle.ts::claimWorkerPidFile startup race (and was killed as a duplicate) or was already stopped and superseded by a newer daemon can never clobber the *current* owner's pid file on its own delayed exit. */
// Upper bound on how long a SIGTERM'd daemon waits for in-flight embedding calls to settle before exiting anyway. `stopWorker` SIGTERMs a running daemon (`worker stop`, or ensureWorkerAlive retiring one that runs a superseded bundle) and expects it gone, but the handler does not stop runWorkerLoop -- an unbounded wait here would let a wedged embedding call (a hung model download, a stuck onnxruntime call) keep that daemon alive and draining after it was told to stop, trading one failure mode (dropped in-flight embeddings) for a worse one (a daemon that cannot be stopped). Bounded, not zero: this is strictly better than the previous `process.exit(0)`, which drained nothing regardless of how fast the in-flight work actually was.
const SIGTERM_DRAIN_TIMEOUT_MS = 5_000
/** Races {@link pendingEmbeddings} against `timeoutMs`, resolving as soon as either settles. Extracted from {@link runDetachedWorkerDaemon}'s SIGTERM handler so a test can drive it directly with a short bound: `pendingEmbeddings()` itself is never mocked here (that would be exactly the injected-seam trap CLAUDE.md's critical-path note warns this file has shipped broken behind before), only how long this function is willing to wait for it. */
export function sigtermDrainDeadline(timeoutMs: number = SIGTERM_DRAIN_TIMEOUT_MS): Promise<void> {
const timeout = new Promise<void>((resolve) => {
setTimeout(resolve, timeoutMs).unref()
})
return Promise.race([pendingEmbeddings(), timeout]).then(() => undefined)
}
export function runDetachedWorkerDaemon(options?: { claimSlot: boolean }): void {
// Resolved against the directory this process was started in, before leaving it, so a relative TG_WORKER_DATA_DIR keeps meaning what its author meant.
const dir = options?.claimSlot === true ? dataDir() : path.resolve(process.env['TG_WORKER_DATA_DIR'] ?? dataDir())
// First act: leave the inherited working directory, which startDetachedWorker already avoids but an older client or a hand-run `--worker-daemon` does not. On Windows a working directory is an open handle, and this process lives for hours, so staying put would keep the caller's directory undeletable for all of them. The daemon finds no project through its working directory: every queued path is absolute. The one reader it reaches is loadConfig()'s per-project `.token-goat.toml` lookup, and from an inherited directory that lookup applied whichever project happened to start the daemon to every project it indexed. The temp directory first, as startDetachedWorker chooses it and for its reason; the data directory when TEMP names a directory that is gone.
for (const neutral of [os.tmpdir(), dir]) {
try {
process.chdir(neutral)