Repository navigation
Expand file tree
/
Copy pathcli.ts
More file actions
1147 lines (1089 loc) · 75.2 KB
/
Copy pathcli.ts
File metadata and controls
1147 lines (1089 loc) · 75.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import { Command } from 'commander'
import { attemptedCommandName, suggestForUnknownCommand } from './command_intent.js'
import { renamedFlagsIn, rewriteOptionIntent, suggestForUnknownOption } from './option_intent.js'
import { spawnSync } from 'child_process'
import * as fs from 'fs'
// Type-only imports: erased at compile time, so referencing them here does not eagerly load mcp_server.js (and transitively the whole MCP protocol layer and zod) at CLI startup. The runtime values are lazy-imported only inside cmdMcpServe.
import type { createMcpServer as CreateMcpServerFn } from './mcp_server.js'
import type { StdioServerTransport as StdioServerTransportClass } from './mcp_stdio.js'
import { buildProjectMap, formatProjectMap, mapLookupBytesSaved, MAX_FILES_SCANNED } from './baseline.js'
import { recordStat, savedTokensFromBytes } from './stats.js'
import { _useRichStats } from './stats_report.js'
import { UNTRUSTED_WEB_TAG } from './injection_scan.js'
import { fenceUntrusted } from './untrusted_fence.js'
import { requireTrackedFiles } from './repomap.js'
import { collectWalkIndexFiles, MAX_FILES_SCANNED_FORCED } from './walk_index.js'
import { dataDir, ENV_KEYS, globalDbPath, VERSION } from './constants.js'
import { assetEmbedSha, indexFileSync, indexFileEmbeddings, indexedPathSpellingIsStale, isEmbedFresh, isParseSkipEligible, loadRegexExtractors, maxChunksEmbedSha } from './parser.js'
import { withDeclaredSkipScopeRoot } from './skip_scope.js'
import { deleteFileEmbeddings, embeddingsDepsAvailable, ensureEmbeddingProvenance } from './embeddings.js'
import { pruneUnembeddableChunks } from './embed_backfill.js'
import { foregroundDownloadDeferred, offlineEmbedNotice, WARM_COMMAND } from './embed_preflight.js'
import { rerunWithEnvProxy } from './env_proxy.js'
import { allowReadOnlyIndex, getDb, limitDbWaitsForHook } from './db.js'
import { pruneDeletedFiles, removeFileFromIndex } from './index_prune.js'
import { recordIndexedRoot } from './indexed_roots.js'
import { recordKnownRootThrottled } from './known_roots.js'
import { fingerprintFile } from './fingerprint.js'
import { readPreHashStamp, restampVerifiedUnchanged } from './reconcile.js'
import { getFileEntry } from './index_reader.js'
import { parserFingerprintForLanguage } from './parser_stamp.js'
import { detectLanguageOfFile } from './parser_types.js'
import { isEmbeddableDocument } from './doc_embed_extract.js'
import { displaySafeText, hostPathOfTypedPath, resolveIndexPath, displaySafeJson } from './paths.js'
import { echoedValue, quotedArg } from './hint_suggestion_guard.js'
import { isUnderSystemTemp, resolveProjectRoot } from './project.js'
import { runParallelSearch } from './search/search_cli.js'
import { ALL_CHANNELS, type SearchChannel } from './search/types.js'
import { embedPolicyResolver, runDetachedWorkerDaemon } from './worker.js'
import { ensureWorkerAlive, isWorkerRunning, startDetachedWorker, stopWorkerDetailed, WorkerAlreadyRunningError, workerLeftRunningNote } from './worker_lifecycle.js'
import { enqueueDirtyPathsSafe } from './hooks_index.js'
// Loaded on demand inside cmdCompress, not at module scope: bash_runner pulls in the whole bash tool-filter registry (every language, linter, cloud and package-manager filter), which only the compress command ever uses. See the same reasoning for relay in cmdHook.
import { runPrSlice } from './read_commands.js'
import { runSemantic, runSemanticMulti } from './read_semantic.js'
import { runSemanticDistances } from './semantic_distances.js'
import {
runExitText,
requireInt,
requireNonNegativeInt,
requirePositiveInt,
} from './cli_dispatch.js'
import { generateCompactHelp } from './cli_help.js'
import { registerInspectCommands } from './cli_cmd_inspect.js'
import { registerIndexCommands } from './cli_cmd_index.js'
import { registerContentCommands } from './cli_cmd_content.js'
import { registerIntegrationCommands } from './cli_cmd_integrations.js'
import { registerDiagnosticCommands } from './cli_cmd_diagnostics.js'
import { registerFormatCommands } from './cli_cmd_formats.js'
import { registerAnalysisCommands } from './cli_cmd_analysis.js'
import { registerSessionCommands } from './cli_cmd_session.js'
import { registerDocumentCommands } from './cli_cmd_documents.js'
import { registerSystemCommands } from './cli_cmd_system.js'
import { cmdHookServerRun, cmdHookServerStatus, cmdHookServerStop } from './cli_hook_server.js'
import { cmdInstall, cmdMcpStatus, cmdUninstall } from './cli_install.js'
import { runPayloads } from './cli_payloads.js'
import { _applyFiltersAndPrint, cmdBashOutput, cmdMcpOutput, cmdWebOutput, type RecallFilterOpts } from './cli_cached_output.js'
import { redactIfDotenv } from './dotenv_redact.js'
import { BRIDGE_CAPABILITY_MATRIX, bridgesStatusToJson, formatBridgesStatus } from './bridges_status.js'
import { buildCommandManifest, filterCommandManifest, formatCommandManifest } from './cli_commands.js'
import { ensureNewline, extractErrorMessage, cappedSourceBytesSaved, isUnderBlockedRoot, countNoun } from './util.js'
import { findProject } from './project.js'
import { colorStdout, stripAnsiEscapes } from './render/ansi.js'
import { loadConfig, getLastConfigParseError, getLastProjectConfigParseError, lastProjectConfigLockedKeys, withConfigProjectRoot } from './config.js'
import { COMPRESSION_PROFILES, parseCompressionProfile } from './config_types.js'
import { applyIndexingPriority } from './process_priority.js'
import { runStats } from './cli_stats.js'
import { runDoctorAndExit, runDoctorChecks } from './cli_doctor.js'
import { runBenchCommand } from './cli_bench.js'
import { expandGlobs } from './cli_diagnostics.js'
export { expandGlobs }
import { emitErr } from './emit.js'
import { compressText, createHandoff, describeContentNamedByHandoff, describeMissingHandoff, describeUnreadableContent, isContentExpired, namedContentNotes, olderCopyNote, resolveHandoffDetailed, retrieveText, unreadableRecordNote, CONTENT_MAX_INPUT_CHARS } from './content_store.js'
import { CliError, clearCliInvocation, formatCommandError, formatFailedResultText, formatParseError, setCliInvocation } from './command_error.js'
// Defined in a leaf so command modules cli.ts imports can render an error the same way without importing cli.ts back; re-exported here for the many modules that already take CliError from cli.js.
export { CliError, formatCommandError }
function stdoutPayload(text: string): string {
const payload = colorStdout() ? text : stripAnsiEscapes(text)
return ensureNewline(payload)
}
export function out(text: string): void {
process.stdout.write(stdoutPayload(text))
}
export function outWithPayload(text: string, observe: (payload: string) => void): void {
const payload = stdoutPayload(text)
observe(payload)
process.stdout.write(payload)
}
export function err(text: string): void {
process.stderr.write(ensureNewline(text))
}
function readBoundedText(text: string | undefined, file: string | undefined): string {
if (text !== undefined && file !== undefined) throw new CliError('provide text or --file, not both')
if (text === undefined && file === undefined) throw new CliError('provide text or --file')
let value: string
if (file !== undefined) {
if (fs.statSync(file).size > CONTENT_MAX_INPUT_CHARS) {
throw new CliError(`file exceeds the ${CONTENT_MAX_INPUT_CHARS}-byte safety limit`)
}
// Masked here, at intake: both stores keep this text and `retrieve` and `handoff-resolve --full` print it back verbatim, and the keyword-driven redactSecrets pass they run misses a dotenv value whose key looks harmless.
value = redactIfDotenv(file, fs.readFileSync(file, 'utf8'))
} else {
if (text === undefined) throw new CliError('provide text or --file')
value = text
}
if (value.length > CONTENT_MAX_INPUT_CHARS) {
throw new CliError(`text exceeds the ${CONTENT_MAX_INPUT_CHARS}-character safety limit`)
}
return value
}
function formatCompression(result: ReturnType<typeof compressText>, forcePayload = false, withheldNotice = 'payload withheld: inlining it would cost more tokens than the original text; pass --payload to print it anyway'): string {
const lines = [
`id: ${result.id}`,
`encoding: ${result.encoding}`,
`original_bytes: ${result.originalBytes}`,
`compact_bytes: ${result.compactBytes}`,
`bytes_saved: ${result.bytesSaved}`,
`tokens_saved: ${result.tokensSaved}`,
`recovery: ${result.recovery}`,
]
// The payload is base64url and tokenizes far worse per byte than the source text, so inlining it usually costs more tokens than it saves; print it only when it genuinely wins or the caller asked for it explicitly.
if (!result.inlineWins && !forcePayload) {
lines.push(withheldNotice)
return lines.join('\n')
}
lines.push('payload:', result.compact)
return lines.join('\n')
}
function cmdContentCompress(text: string | undefined, opts: { file?: string; payload?: boolean }): void {
out(formatCompression(compressText(readBoundedText(text, opts.file)), opts.payload === true))
}
function cmdRetrieve(id: string, opts: RecallFilterOpts): void {
const text = retrieveText(id)
if (text === null) throw new CliError(describeUnreadableContent(id) ?? describeContentNamedByHandoff(id) ?? (isContentExpired(id) ? `content ${id} expired: it is older than the local cache's age limit.` : `no token-goat content for id: ${id}. The local cache may have expired.`))
// retrieve is the lossless round-trip for compress-text; other commands print `recovery: token-goat retrieve <id>` promising the original bytes back, so with no narrowing flag it must stay byte-verbatim -- only opt into sibling head/tail elision once the caller explicitly asks for a slice.
const noNarrowing = opts.head === undefined && opts.tail === undefined && opts.grep === undefined && opts.section === undefined && opts.maxMatches === undefined && opts.lines === undefined && opts.context === undefined
// Written raw, not through out() or the shared filter: both strip ANSI codes for a non-colour stdout and the filter re-joins CRLF lines with LF, so the "original bytes back" came back altered whenever the stored text had either. No trailing newline is added either, so `retrieve <id> > file` reproduces the file compress-text read.
if (noNarrowing && opts.lineNumbers !== true) {
process.stdout.write(text)
return
}
_applyFiltersAndPrint(text, noNarrowing ? { ...opts, full: true } : opts)
}
function cmdHandoffCreate(name: string, text: string | undefined, opts: { file?: string }): void {
const result = createHandoff(name, readBoundedText(text, opts.file))
// On stderr so stdout stays exactly the record; the exit code is unchanged.
for (const note of namedContentNotes(result)) emitErr(note)
out(displaySafeJson(result))
}
function cmdHandoffResolve(name: string, opts: { full?: boolean }): void {
const resolved = resolveHandoffDetailed(name, { full: opts.full === true })
if (resolved === null) throw new CliError(describeMissingHandoff(name))
const { result, olderCopies, unreadable } = resolved
// On stderr so stdout stays exactly the handoff; the newest differing older copy is the one worth reading, and its content id is what `retrieve` takes.
const note = olderCopyNote(name, olderCopies, (contentId) => `token-goat retrieve ${quotedArg(contentId)}`)
if (note !== null) emitErr(note)
const unreadableNote = unreadableRecordNote(name, unreadable)
if (unreadableNote !== null) emitErr(unreadableNote)
out(typeof result === 'string' ? result : formatCompression(result, false, 'payload withheld: inlining it would cost more tokens than the original text; pass --full to get the original text back outright'))
}
// The numeric CLI-flag validators live in cli_dispatch.ts; re-exported here because half the command modules import them from this path and half from that one, and one definition behind both spellings is what keeps the two from drifting.
export { requireInt, requireNonNegativeInt, requirePositiveInt }
// --- Command handlers -------------------------------------------------------
// Thin wrapper: all orchestration (embedding search, merge, FTS fallback, formatting) lives in read_semantic.ts's runSemantic so the MCP server (mcp_server.ts) can call the same logic in-process without going through the CLI/commander layer.
async function cmdSemantic(query: string | undefined, more: string[], opts: { limit?: string; json?: boolean; grep?: string; excludeTests?: boolean; preflight?: boolean; warm?: boolean; distances?: boolean; all?: boolean }): Promise<void> {
if (opts.distances === true) {
if (query !== undefined || opts.preflight === true || opts.warm === true) throw new CliError('--distances reports recorded queries and runs none; drop the query, --preflight and --warm')
runExitText(() => runSemanticDistances({ ...(opts.all === true ? { all: true } : {}) }))
return
}
if (!query && !opts.preflight && !opts.warm) {
throw new CliError('missing required argument: query')
}
// Rejected rather than ignored, as `symbol` rejects --grep with a name: --preflight never searches, so several queries beside it can only be a mistake.
if (opts.preflight === true && more.length > 0) {
throw new CliError('--preflight checks the embedding setup and runs no query; drop the queries, or drop --preflight to search')
}
// --warm is a download the user asked for now, so on a machine whose proxy this process's fetch would go around, the command runs again in a child that goes through it (env_proxy.ts rerunWithEnvProxy).
if (opts.warm === true) {
const code = rerunWithEnvProxy(spawnSync)
if (code !== null) {
process.exitCode = code
return
}
}
// Parsed as any integer here so runSemantic refuses 0 and a negative value itself, in the same words, where --json can put the refusal on stdout as JSON.
const limit = opts.limit !== undefined ? requireInt('--limit', opts.limit) : 20
const semanticOpts = {
limit,
...(opts.json === true ? { json: true } : {}),
...(opts.grep !== undefined ? { grep: opts.grep } : {}),
...(opts.excludeTests === true ? { excludeTests: true } : {}),
...(opts.preflight === true ? { preflight: true } : {}),
...(opts.warm === true ? { warm: true } : {}),
}
const { text, code } = more.length > 0 ? await runSemanticMulti([query ?? '', ...more], semanticOpts) : await runSemantic(query ?? '', semanticOpts)
// --json must always land on stdout so `| jq .` works even on a no-match/error exit -- only the text-mode path routes a non-zero code to stderr, as a command error.
if (opts.json === true || code === 0) out(text)
else err(formatFailedResultText(text))
process.exitCode = code
}
async function cmdSearch(
query: string | undefined,
more: string[],
opts: {
limit?: string
channels?: string
project?: string | boolean
json?: boolean
minScore?: string
},
): Promise<void> {
const fullQuery = [query, ...more].filter(Boolean).join(' ').trim()
if (!fullQuery) {
throw new CliError('missing required argument: query')
}
let projectRoot: string | undefined
if (opts.project === true) {
projectRoot = resolveProjectRoot({ project: process.cwd() })
} else if (typeof opts.project === 'string') {
projectRoot = resolveProjectRoot({ project: opts.project })
}
const limit = opts.limit !== undefined ? requirePositiveInt('--limit', opts.limit) : 20
let channels: SearchChannel[] | undefined
if (opts.channels) {
const rawList = opts.channels.split(',').map((c) => c.trim().toLowerCase()).filter(Boolean)
for (const c of rawList) {
if (!ALL_CHANNELS.includes(c as SearchChannel)) {
throw new CliError(`Unknown search channel ${echoedValue(c)}. Supported channels: ${ALL_CHANNELS.join(', ')}`)
}
}
channels = Array.from(new Set(rawList)) as SearchChannel[]
}
let minScore: number | undefined
if (opts.minScore !== undefined) {
const rawVal = opts.minScore.trim()
const parsed = Number(rawVal)
if (rawVal === '' || !Number.isFinite(parsed) || parsed < 0) {
throw new CliError(`Invalid --min-score: ${echoedValue(opts.minScore)} (must be a non-negative number)`)
}
minScore = parsed
}
const { text, code } = await runParallelSearch({
query: fullQuery,
channels,
limit,
projectRoot,
json: opts.json === true,
minScore,
})
if (opts.json === true || code === 0) out(text)
else err(formatFailedResultText(text))
process.exitCode = code
}
/** How long `index` spends embedding in the foreground, after the first file, before it leaves the rest to the background worker. */
export const INDEX_INLINE_EMBED_BUDGET_MS = 20_000
export async function cmdIndex(
pathArg?: string,
opts: { walk?: boolean; dbPath?: string; force?: boolean; forceWalk?: boolean; embed?: boolean; embedBudgetMs?: number } = {},
): Promise<void> {
// A typed root can be spelled at a drive mount (`/mnt/c/x` or `/c/x` reaching a Windows process unconverted, `C:\x` under WSL). hostPathOfTypedPath opens it where this host keeps it; used as typed, the mount named a folder that does not exist here and the run reported `Indexed 0 files` as a success.
const root = pathArg === undefined ? process.cwd() : hostPathOfTypedPath(pathArg)
if (!fs.existsSync(root)) throw new CliError(`${echoedValue(pathArg ?? root)} does not exist.`)
const dbPath = opts.dbPath ?? globalDbPath()
const force = opts.force === true
const useWalk = opts.walk === true || opts.forceWalk === true
let files = requireTrackedFiles(root)
if (files.length === 0) {
if (!useWalk) {
throw new CliError(
`no tracked files found under ${echoedValue(root)} (is it a git repo?). ` +
`Pass --walk or --force-walk to index a non-git folder.`,
)
}
// Opt-in non-git fallback: a bounded directory walk, guarded against over-broad roots / oversized trees and stripped of .env / generated files.
files = collectWalkIndexFiles(root, { force: opts.forceWalk === true })
if (opts.forceWalk === true) {
process.stderr.write(
`token-goat: --force-walk raised the walk cap to ${MAX_FILES_SCANNED_FORCED} files; ` +
`indexing ${countNoun(files.length, 'file')} may take a long time and produce a large index. ` +
`Run \`token-goat doctor\` afterwards to check index size.\n`,
)
}
}
// A bulk walk is long-running background work even though the user typed it: they start it and go back to their editor. The daemon lowers its own priority for the same reason; doing it here too is what makes "both indexing paths" true rather than only the invisible one. It happens only now, after the file list exists, and not on entry: `git ls-files` is a child that inherits the priority of the process that spawns it, and it does nothing but wait on the disk, so demoting first made it queue behind every busy process on the machine (1.6 s became 11.3 s under load, minutes on a real install) without sparing the user any CPU. The parse and embed work below is what the demotion is for.
applyIndexingPriority()
const blockedRoots = loadConfig().worker.blocked_roots
const ixCfg = loadConfig().indexing
// The root whose segments indexing.skip_dirs is tested against: a project kept under an ancestor named `build` must not read as vendored. Spelled like the keys it is compared with.
const walkRootKey = resolveIndexPath(root)
// Whether a file is embedded is decided by its own project's configuration, through the rule the worker's drain applies to the same file (see embedPolicyResolver), never by the directory this command runs in: run inside a monorepo package, or given a path from elsewhere, the index and the drain stamped one file two ways and each undid the other. The other indexing keys this function reads through loadConfig() with no root are right from any root only because each is in PROJECT_LOCKED_KEYS, so no project file can set one.
const embedPolicy = embedPolicyResolver()
let indexed = 0
let embedsDeferred = 0
let embedsDeferredNoWorker = 0
// Embedding runs at roughly ten chunks a second on the bundled WebAssembly runtime, so embedding a repository of a few thousand files inline kept this command running for hours. Once the inline embeds have used their time budget, the rest are put on the worker's dirty queue, the path `install` already uses. The first embed is not counted because it includes loading, and on a first run downloading, the model. --embed waits for every file here instead.
const embedBudgetMs = opts.embed === true ? Number.POSITIVE_INFINITY : (opts.embedBudgetMs ?? INDEX_INLINE_EMBED_BUDGET_MS)
let embedsInline = 0
let embedSpentMs = 0
let embedsBackground = 0
// Every file left for the worker to embed, deferred or past the budget, put on its dirty queue once the walk is done.
const leftForWorker: string[] = []
let failed = 0
let skipped = 0
const failureGroups = new Map<string, { example: string; count: number }>()
// Manual `index` runs can take minutes on a real repo with nothing printed until the very end, which looks hung on a real terminal but must stay perfectly silent for pipes/CI/hook invocations that parse stdout -- reuse _useRichStats' exact TTY/NO_COLOR/FORCE_COLOR gate so the same rule that governs rich stats output governs this progress line. Progress is written to stderr only and throttled to ~10 repaints/sec so a large repo does not hammer the terminal with one line per file.
const showProgress = _useRichStats()
const progressStart = Date.now()
let lastProgressPaintAt = 0
let lastProgressLineLen = 0
const totalFiles = files.length
let fileIdx = 0
function paintProgress(phase: string): void {
if (!showProgress) return
const now = Date.now()
if (now - lastProgressPaintAt < 100) return
lastProgressPaintAt = now
const elapsedSec = ((now - progressStart) / 1000).toFixed(1)
const text = `${fileIdx}/${totalFiles} files -- ${phase} -- ${elapsedSec}s elapsed`
const trailingPad = text.length < lastProgressLineLen ? ' '.repeat(lastProgressLineLen - text.length) : ''
lastProgressLineLen = text.length
process.stderr.write(`\r${text}${trailingPad}`)
}
for (const f of files) {
fileIdx += 1
paintProgress('scanning')
// Key on the same canonical absolute-normalized path every reader resolves to via resolveIndexPath. requireTrackedFiles returns path.join(root, rel), so a relative root (the natural `token-goat index .`) yields relative paths; normalizePath alone would store a relative key that no reader can match.
const key = resolveIndexPath(f)
// worker.blocked_roots (set via `token-goat project exclude`) excludes a path prefix from indexing entirely -- skip before the language check so a blocked file is never touched.
if (isUnderBlockedRoot(key, blockedRoots)) {
// Purge rather than skip. A file indexed before its root was blocked would otherwise keep its symbols, bodies and embeddings forever: a plain skip leaves the rows it wrote behind, and no other pass removes them (pruning is existence-based, and an excluded file is still on disk). Same treatment isParseSkipEligible already gives a file excluded by skip_dirs.
removeFileFromIndex(getDb(dbPath), key)
continue
}
// PDF/DOCX/PPTX/XLSX have no Language entry (no code symbols) so they report 'unknown', but they must still reach indexFileEmbeddings below for extracted-text embedding. The language is read from the file's head, as the walk that listed it did: a path-only check skipped every `.p`, `.w`, `.m` and `.t` a content sniff admits, so it was never indexed or counted.
if (detectLanguageOfFile(key) === 'unknown' && !isEmbeddableDocument(key)) continue
// indexing.skip_dirs / large_file_skip_kb: filter here, before the sha/entry work below. Without this pre-filter, indexFileSync's internal purge would run and then the unconditional indexFileEmbeddings call below would immediately re-embed a file meant to be fully excluded (origin's indexFileEmbeddings has no skip_dirs/size-cap branch).
if (isParseSkipEligible(key, ixCfg, walkRootKey)) {
removeFileFromIndex(getDb(dbPath), key)
continue
}
// Stat and clock before the hash, so a clean proof below is recorded under the mtime the hashed bytes carried and an index time no later than the hash (see readPreHashStamp).
const stamp = readPreHashStamp(key)
// Mirror worker.ts's makeIndexer sha gate here: a bulk `token-goat index` run previously called indexFileSync (and re-chunked/re-embedded via indexFileEmbeddings) unconditionally for every tracked file on every invocation, even ones byte-identical to what was already indexed. fingerprintFile returning null (a transient read failure/race) is treated as "not unchanged" so the file still gets a normal reindex attempt below. Parse and embed freshness are gated independently (embed_sha vs sha), matching makeIndexer, so a file whose embedding previously failed still gets re-embedded even when its parse is current. --force bypasses both freshness checks unconditionally -- e.g. after a parser.ts extraction-logic change, every already-indexed file's SHA is untouched and stale symbols/refs would otherwise never get recomputed until each file happens to be edited.
const sha = fingerprintFile(key)
// A git-tracked file deleted from the worktree is still listed by requireTrackedFiles, so it reaches this loop on every run. fingerprintFile returns null for it, indexFileSync fail-softs on ENOENT without throwing, and the `indexed += 1` at the bottom of the loop then counted work that never happened -- every run, forever, since deleting the file is exactly what keeps it in this state. After a rename the effect was the headline symptom: `Indexed 1 file into the symbol index` printed while the index had just been emptied. A null sha for a file that DOES exist is a transient read failure (a lock held by an AV scanner or an open editor) and still deserves the normal reindex attempt below, so the existence check is what separates the two. The rows are removed by pruneDeletedFiles after the loop, which is the pass that owns vanished files.
if (sha === null && !fs.existsSync(key)) continue
// Register the project root as sweepable before the freshness gates below can skip this file: a walk that finds everything already current still proves the project has rows worth sweeping. See recordKnownRootThrottled.
recordKnownRootThrottled(key, dataDir(), dbPath)
const { configRoot, embeddingsEnabled } = embedPolicy(key)
// See isEmbedFresh: depsAvailable keeps an `unavailable:`-marked embed_sha (a file skipped only because the optional model/sqlite-vec deps were absent) treated as stale so it is re-embedded once the deps are installed, instead of looking permanently fresh.
const depsAvailable = embeddingsEnabled && embeddingsDepsAvailable(getDb(dbPath))
if (depsAvailable) {
// files.embed_sha records WHICH CONTENT was embedded, never WHICH STACK embedded it, so the per-file freshness gate below cannot see a model or inference-runtime change on its own: it reads a bare sha as fresh and skips the file. ensureEmbeddingProvenance owns that input and is the only thing that can re-open the decision, but its only callers were upsertChunks and searchSemantic, both downstream of that gate -- so a whole-index run after an onnxruntime major.minor upgrade printed "Skipped N unchanged file(s)" and left every vector from the previous stack in place, which is exactly what the warning that reset prints tells the user to run this command to fix. It must run before the row read below, not beside the gate that consults the row: the reset clears each affected file's embed_sha, and once `entry` holds a row that clearing can no longer be seen. Memoized per database per process, so after the first file this costs one Set lookup. Gated on the deps being usable because backendId() cannot name a runtime that did not load, and wiping the index on the strength of an unknowable identity would be worse than the staleness it is guarding against.
ensureEmbeddingProvenance(getDb(dbPath))
// Chunk rows written before the asset and chunk-count gates existed carry a valid embed_sha, so the per-file freshness gate below reads every one of them as current and would leave them searchable forever. Same placement and the same reason as ensureEmbeddingProvenance directly above: it has to run before a row is read. See src/embed_backfill.ts for why this is a version-keyed sweep rather than a fingerprint bump.
pruneUnembeddableChunks(getDb(dbPath), ixCfg.max_chunks_per_file, deleteFileEmbeddings, { asset: assetEmbedSha, maxChunks: maxChunksEmbedSha })
}
const entry = sha !== null ? getFileEntry(key, dbPath) : null
// A case-only rename (`mv b.ts B.ts`) leaves the content byte-identical, so the sha gate below would skip the file and the row would keep the old spelling indefinitely -- see indexedPathSpellingIsStale. Reindexing rewrites the row under the spelling the file actually has.
const spellingStale = entry !== null && indexedPathSpellingIsStale(entry.filePath, key)
// entry.parserSha gates on WHICH parser wrote the rows, not just whether the content moved -- see parserFingerprintForLanguage and the same gate in worker.ts's makeIndexer. The expected stamp is the one for the language the row itself records, so a fix to one adapter reparses that language's files and leaves every other language's rows alone. This is what makes the --force escape hatch described above unnecessary after a parser change: the mismatch reparses the file on its own.
const parseUnchanged =
!force &&
!spellingStale &&
sha !== null &&
entry?.sha === sha &&
entry.parserSha === parserFingerprintForLanguage(entry.language)
// The hash just proved this file's bytes are the ones the row holds, so a racy row is stamped here as the worker's drain does; skipping the file without it left the row racy, and every later sweep hashed the file again.
if (parseUnchanged && stamp !== undefined) restampVerifiedUnchanged(dbPath, [{ indexedPath: entry.filePath, sha, stamp }])
// isEmbedFresh (parser.ts) is the shared read side of this gate, also used by worker.ts's makeIndexer: while embeddings are config-disabled, only the `disabled:` marker for this sha counts as fresh; while enabled, a bare sha match is fresh (the file was really embedded, or was empty / permanently policy-skipped -- e.g. profile-meta.xml, an oversized salesforce_metadata file -- with nothing to embed, both terminal regardless of deps); and an `unavailable:` marker is fresh only while the optional embedding deps stay uninstalled. Embed freshness is decided on its own inputs, never on parseUnchanged: files.parser_sha answers "which extractor wrote the symbol rows", which carries no information about whether the stored vectors match this content, so conjoining it made a parser-stamp bump re-embed the whole index (measured: a stamp-only reparse of 300 unchanged files cost 94% of indexing them from nothing) and made writeParseResult's embedShaToCarry dead for the waste it exists to prevent. isEmbedFresh already answers false for a new file (no stored embed_sha) and for moved content (the stored sha no longer matches), so the sha coupling was redundant -- except for spellingStale, which is kept by name: a case-only rename leaves the content byte-identical, so isEmbedFresh would say fresh, but `chunks` rows are keyed by file_path and would keep the old spelling forever.
const embedFreshFor = (embedSha: string | undefined): boolean =>
!force &&
!spellingStale &&
sha !== null &&
isEmbedFresh(
embedSha,
sha,
embeddingsEnabled,
depsAvailable,
// See isEmbedFresh: an `oversize:` marker stays fresh only while indexing.large_file_symbol_only_kb is still what it was stamped under, so raising the threshold re-embeds the files it just admitted instead of leaving them permanently skipped. 0 matches no marker (config floors this key at 1), the safe direction for a partially-mocked config.
loadConfig().indexing?.large_file_symbol_only_kb ?? 0,
// Same reasoning and same 0 fallback for the chunk-count marker.
loadConfig().indexing?.max_chunks_per_file ?? 0,
)
const embedUnchanged = embedFreshFor(entry?.embedSha)
if (parseUnchanged && embedUnchanged) {
skipped += 1
continue
}
if (!parseUnchanged) {
paintProgress('parsing')
try {
withDeclaredSkipScopeRoot(walkRootKey, () => indexFileSync(key, dbPath))
} catch (e) {
// A single locked/permission-denied file (AV scan, open editor, OneDrive sync -- all common on Windows) must not abort the rest of a bulk walk. indexFileSync itself only fail-softs on ENOENT (the file vanished between discovery and read, a benign race) and rethrows everything else so callers can report it -- worker.ts's makeIndexer already catches and logs that per-file via an INDEX_FAILED sentinel, but this foreground loop had no try/catch at all, so the same rethrow aborted the whole command uncaught.
failed += 1
const message = extractErrorMessage(e)
const group = failureGroups.get(message)
if (group !== undefined) {
group.count += 1
} else {
failureGroups.set(message, { example: key, count: 1 })
}
continue
}
// indexFileSync fail-softs on ENOENT, so a file deleted during its own parse leaves nothing written and throws nothing either. Counting it would reintroduce the phantom credit the pre-parse guard above exists to stop, just through a narrower window.
if (!fs.existsSync(key)) continue
}
// Re-read the stamp the parse above just wrote rather than trusting the one captured before it: writeParseResult clears the carried embed_sha when a reparse moved this file's embedding boundaries (see embeddingBoundariesMoved), and `embedUnchanged` was computed from the pre-parse row. Without this the re-embed is deferred to whatever run happens next, so a single `token-goat index` after an adapter change leaves the file's vectors cut on boundaries that no longer exist.
const embedFresh = parseUnchanged ? embedUnchanged : embedFreshFor(getFileEntry(key, dbPath)?.embedSha)
// The worker's dirty queue and its backlog sweep both pass over a path under the OS temp dir (see isUnderSystemTemp), so a file there that this run leaves unembedded stays out of `semantic` for good. Such a file is embedded here whatever the budget, and when only the worker could download the model it is reported without promising that the worker will embed it.
const workerWillEmbed = !embedFresh && depsAvailable && !isUnderSystemTemp(key)
// A model that is not on this machine and cannot be fetched now (offline, the last download failed recently, or this process would go around the machine's proxy to get it) makes every embed below fail the same way, one file at a time. Skipping leaves embed_sha unset, so the worker, or the next run, embeds the file once the download succeeds. Only a file that would really download is deferred: with embeddings off, or the embedding packages absent, the call below writes the terminal marker that keeps the file from being retried, and that needs no download.
if (!embedFresh && depsAvailable && foregroundDownloadDeferred()) {
if (workerWillEmbed) {
embedsDeferred += 1
leftForWorker.push(key)
} else {
embedsDeferredNoWorker += 1
}
} else if (workerWillEmbed && embedsInline > 0 && embedSpentMs >= embedBudgetMs) {
// Left with embed_sha unset and handed to the worker below. Only a file that would really embed is handed over: with embeddings off or their packages absent, the call below writes a terminal marker in no time and must still run.
embedsBackground += 1
leftForWorker.push(key)
} else if (!embedFresh) {
paintProgress('embedding')
const embedStart = Date.now()
// Best-effort semantic-embeddings step for the same file, run right after its syntactic parse; awaited here because this is a one-shot foreground command the caller waits on, unlike the worker's incremental drain which fires this and forgets it. Passing sha lets it stamp files.embed_sha on success, the same embed-freshness gate makeIndexer uses. It reads `embeddings_enabled` with no root to pass, as its first synchronous statement, so the file's own project is the one loadConfig() resolves for that stretch, as in worker.ts's embedFileSerialized.
await withConfigProjectRoot(configRoot, () => indexFileEmbeddings(key, dbPath, sha ?? undefined))
if (depsAvailable) {
if (embedsInline > 0) embedSpentMs += Date.now() - embedStart
embedsInline += 1
}
}
indexed += 1
}
// Clear the progress line before any further stderr writes (failure summaries below) or the final stdout summary, so nothing is left overwritten or trailing on the terminal.
if (showProgress && lastProgressLineLen > 0) {
process.stderr.write(`\r${' '.repeat(lastProgressLineLen)}\r`)
}
for (const [message, group] of failureGroups) {
err(
`token-goat: index: failed to index ${echoedValue(group.example)}: ${message}` +
(group.count > 1 ? ` (and ${countNoun(group.count - 1, 'other file')})` : ''),
)
}
const pruned = pruneDeletedFiles(resolveIndexPath(root), dbPath)
// What session start and install read as "this project is indexed": the symbols this walk wrote cannot say it, because the edit hook writes symbols for single files too.
recordIndexedRoot(root, dbPath)
out(
`Indexed ${countNoun(indexed, 'file')} into the symbol index.` +
`${skipped > 0 ? ` Skipped ${countNoun(skipped, 'unchanged file')}.` : ''}` +
`${pruned > 0 ? ` Pruned ${countNoun(pruned, 'deleted file')}.` : ''}` +
`${failed > 0 ? ` Failed to index ${countNoun(failed, 'file')} (see stderr).` : ''}`,
)
// The files left without an embed go on the worker's dirty queue, which wakes a worker that is running and starts one that is not; the worker is also the process whose downloads go through the proxy. Leaving them to its backlog sweep was not enough: the sweep walks the index once per worker start, so a worker an earlier hook started had finished that walk before this run began, and the files stayed out of `semantic` until it next restarted.
if (leftForWorker.length > 0) enqueueDirtyPathsSafe(leftForWorker, { alreadyResolved: true })
if (embedsDeferred > 0) {
ensureWorkerAlive()
const offlineNotice = offlineEmbedNotice(countNoun(embedsDeferred, 'file'), root)
if (offlineNotice !== null) err(offlineNotice)
else err(`token-goat: index: ${countNoun(embedsDeferred, 'file')} not embedded yet: the embedding model is not downloaded. The background worker downloads it and embeds them; to do it now, run \`${WARM_COMMAND}\` and then \`token-goat index\` again.`)
}
if (embedsDeferredNoWorker > 0) {
const files = countNoun(embedsDeferredNoWorker, 'file')
err(offlineEmbedNotice(files, root) ?? `token-goat: index: ${files} not embedded: the embedding model is not downloaded, and the background worker does not embed files under the system temp directory. To embed them, run \`${WARM_COMMAND}\` and then \`token-goat index\` again.`)
}
if (embedsBackground > 0) {
ensureWorkerAlive()
err(`token-goat: index: ${countNoun(embedsBackground, 'file')} left for the background worker to embed; \`semantic\` uses keyword search for them until it finishes. To embed them here instead, run \`token-goat index --embed\`.`)
}
// A run where every file failed and none indexed is a total indexing failure, not a no-op success -- callers scripting on `$?` must be able to detect it.
if (indexed === 0 && failed > 0) {
process.exitCode = 1
}
}
function cmdMap(opts: { compact?: boolean; json?: boolean }): void {
const map = buildProjectMap(process.cwd(), { compact: opts.compact === true, emitsJson: opts.json === true })
const text = formatProjectMap(map, map.compact)
if (opts.json === true) {
out(displaySafeJson(map, 0))
} else {
out(text)
}
// `map_lookup` has carried a live entry in stats.ts's KIND_TO_SOURCE/COMMAND_KINDS registry since the Python->TS port, but nothing ever called recordStat for it -- the `map`/`baseline` dashboard bucket was permanently zero regardless of real usage (same class of gap fixed for changed_lookup, see project_runchanged_missing_stat memory). The byte accounting (including the recentFiles-vs-topSymbols path canonicalization needed for the dedup) lives in mapLookupBytesSaved so cmdMap and the MCP `map` tool share one implementation.
const bytesSaved = mapLookupBytesSaved(map, text)
recordStat('map_lookup', bytesSaved, savedTokensFromBytes(bytesSaved))
}
function cmdBridgesStatus(opts: { json?: boolean }): void {
if (opts.json === true) {
out(displaySafeJson(bridgesStatusToJson(BRIDGE_CAPABILITY_MATRIX), 0))
} else {
out(formatBridgesStatus(BRIDGE_CAPABILITY_MATRIX))
}
}
function cmdCommands(opts: { json?: boolean; grep?: string }): void {
let manifest = buildCommandManifest(buildProgram())
if (opts.grep !== undefined) {
manifest = filterCommandManifest(manifest, opts.grep)
}
if (opts.json === true) {
out(displaySafeJson(manifest, 0))
} else if (manifest.length === 0) {
// Same wording as cmdPptxText's --grep-with-no-hits path: a filter matching nothing is a legitimate empty result, not an error, so this stays a plain message on exit 0.
out('no matches')
} else {
out(formatCommandManifest(manifest))
}
}
// Runs an MCP stdio server exposing read/symbol/section/outline/skeleton/semantic as tools. The returned promise only resolves once the server reports its connection closed (via `onclose`, set after `connect()` so it's not clobbered by the wiring `connect()` itself does to the transport's own `onclose`) -- resolving early here would let `run()`'s caller (main.ts) return while the process still has useful work queued on stdin.
async function cmdMcpServe(): Promise<void> {
let createMcpServer: typeof CreateMcpServerFn
let StdioServerTransport: typeof StdioServerTransportClass
// Both modules are token-goat's own and are inlined into the bundle, so unlike the days when this loaded an optional third-party SDK there is no "not installed" case left. The guard stays because a dynamic import can still fail on a truncated or partially written install, and a bare rejection here would surface as an unhandled crash rather than a command that reports what went wrong.
try {
;({ createMcpServer } = await import('./mcp_server.js'))
;({ StdioServerTransport } = await import('./mcp_stdio.js'))
} catch (err) {
process.stderr.write(formatCommandError(`mcp-server unavailable (could not load the MCP server modules): ${String(err)}`) + '\n')
process.exitCode = 1
return
}
const server = await createMcpServer()
const transport = new StdioServerTransport()
await server.connect(transport)
await new Promise<void>((resolve) => {
server.onclose = resolve
})
}
async function cmdHook(event: string, opts: { harness?: string }): Promise<void> {
// A bridge that writes a bare command string into its host tool's config (no in-process env-setting hook like pi.ts/copilot_cli.ts have) can self-identify via this flag instead — same purpose as TOKEN_GOAT_HARNESS_OVERRIDE, just passed as an argv flag since there's no JS relay script in the middle to set process.env directly. detectHarness() itself already validates the value against KNOWN_HARNESS_NAMES and ignores anything unrecognized, so no extra validation is needed here.
if (typeof opts.harness === 'string' && opts.harness.length > 0) {
process.env[ENV_KEYS.HARNESS_OVERRIDE] = opts.harness
}
// Imported here rather than at module scope: relay.ts side-effect-imports every hook handler to register them, so a top-level import made every CLI command -- `symbol`, `read`, even `--version` -- parse the whole hook subsystem, the bash tool-filter registry and the HTML extractor before doing anything. Only this one command needs any of it. Hooks themselves are unaffected: they run through dist/token-goat-hook.mjs, which imports relay directly.
const { relay } = await import('./relay.js')
// A hook is the agent's tool call, so no database lock a wedged process holds may stall it for the 15 s the worker waits.
limitDbWaitsForHook()
// relay handles its own stdin read / stdout write and never throws on a malformed/unknown event — it emits `{}` and returns.
await relay(event)
}
function cmdWorkerStart(): void {
if (isWorkerRunning()) {
out('Worker already running.')
return
}
// startDetachedWorker's own atomic pid-file claim (see worker_lifecycle.ts::claimWorkerPidFile) is the real guard against the TOCTOU race above: two near-simultaneous `worker start` invocations can both pass the isWorkerRunning() check above, but only one of them can win the exclusive pid-file create that follows, so the loser reports this cleanly instead of orphaning a second, unstoppable daemon.
try {
const pid = startDetachedWorker()
out(`Worker started (pid ${pid}).`)
} catch (e) {
if (e instanceof WorkerAlreadyRunningError) {
out('Worker already running.')
return
}
throw e
}
}
function cmdWorkerStop(): void {
const { stopped, leftRunning } = stopWorkerDetailed()
// A stop that left the worker running did not stop it: the CliError makes `guard` print the note on stderr and exit 1.
if (leftRunning !== null) throw new CliError(workerLeftRunningNote(leftRunning))
out(stopped ? 'Worker stopped.' : 'No running worker.')
}
function cmdWorkerStatus(): void {
out(isWorkerRunning() ? 'Worker is running.' : 'Worker is not running.')
}
function cmdStats(opts: { json?: boolean; windowDays?: string; homeDir?: string; full?: boolean; short?: boolean; methodology?: boolean; hooks?: boolean; payloads?: boolean } = {}): void {
if (opts.payloads === true) {
runPayloads(opts.json === true)
return
}
const windowDays = opts.windowDays !== undefined ? requireNonNegativeInt('--window-days', opts.windowDays) : 30
const statsOpts: Parameters<typeof runStats>[0] = {
json: opts.json === true,
windowDays,
full: opts.full === true,
short: opts.short === true,
methodology: opts.methodology === true,
hooks: opts.hooks === true,
}
if (opts.homeDir !== undefined) {
statsOpts.homeDir = opts.homeDir
}
runStats(statsOpts)
}
async function cmdDoctor(opts: { context?: boolean; json?: boolean; repair?: boolean; fix?: boolean; probe?: string; fast?: boolean }): Promise<void> {
if (opts.probe !== undefined) {
// Loaded on demand: a probe spawns a whole harness, and nothing else in doctor needs this module.
const { IDE_HOSTS_WITHOUT_PROBE, NO_HEADLESS_PROBE_EXIT_CODE, PROBE_COMMANDS, formatProbeReport, isProbeHarness, noHeadlessProbeMessage, probeVerdict, runProbe } = await import('./doctor_probe.js')
const harness = opts.probe.trim().toLowerCase()
if (IDE_HOSTS_WITHOUT_PROBE.has(harness)) {
err(`token-goat: ${noHeadlessProbeMessage(harness)}`)
process.exitCode = NO_HEADLESS_PROBE_EXIT_CODE
return
}
if (!isProbeHarness(harness)) {
throw new CliError(`no headless probe for ${echoedValue(opts.probe)}; supported: ${Object.keys(PROBE_COMMANDS).join(', ')}`)
}
const report = runProbe(harness)
out(opts.json === true ? displaySafeJson(report, 0) : formatProbeReport(report))
const verdict = probeVerdict(report)
if (verdict === 'fail') throw new CliError('doctor probe failed')
// Exit 2, not 1: the harness failed before its model answered, so the probe judged no hook either way.
if (verdict === 'inconclusive') process.exitCode = 2
return
}
const doctorOpts: { dataDir?: string; configPath?: string; context?: boolean; rootDir?: string; repair?: boolean; fast?: boolean } = {}
if (opts.fast === true) {
doctorOpts.fast = true
}
if (opts.context === true) {
doctorOpts.context = true
}
if (opts.json === true && (opts.repair === true || opts.fix === true)) {
// The JSON output is a bare array of checks with no place for what a repair did, so asking for both used to run the checks and silently skip the repair.
throw new CliError('doctor --json cannot repair: run `token-goat doctor --fix`, then `token-goat doctor --json` to read the result')
}
if (opts.repair === true || opts.fix === true) {
// A repair downloads what is missing, now, so it runs again through the machine's proxy as `semantic --warm` does.
const code = rerunWithEnvProxy(spawnSync)
if (code !== null) {
process.exitCode = code
return
}
doctorOpts.repair = true
}
// Scope the Symbols check to the invoking project so an unrelated project sharing the same global.db can't mask this project's own parser being broken (see checkSymbolCount's doc comment). No project root found (bare directory, no git/package.json) falls back to the prior unscoped whole-database behavior.
const project = findProject(process.cwd())
if (project !== null) {
doctorOpts.rootDir = project.root
}
if (opts.json === true) {
// --json bypasses printDoctorResults' prose entirely (no `[WARN]`-prefixed lines) and emits the same DoctorResult[] runDoctorChecks() computes, one entry per check with its ok/warn/fail status -- matching cmdCommands'/cmdBridgesStatus' plain JSON.stringify convention (no envelope) rather than inventing a new shape.
const processes = doctorOpts.fast === true ? [] : undefined
const results = await runDoctorChecks(doctorOpts.dataDir, doctorOpts.configPath, doctorOpts.rootDir, processes)
out(displaySafeJson(results, 0))
if (results.some((r) => r.status === 'fail')) {
throw new CliError('doctor checks failed')
}
return
}
const code = await runDoctorAndExit(doctorOpts)
if (code !== 0) {
throw new CliError('doctor checks failed')
}
}
export { fileSizeOrZero } from './cli_office.js'
export { fenceFileText, fenceFileFieldIfMatched } from './untrusted_fence.js'
export { cmdPdfExtract, cmdPdfLocate, cmdPdfOutline, cmdPdfMeta } from './cli_office.js'
export {
cmdDocxOutline,
cmdDocxTables,
cmdDocxText,
cmdPptxNotes,
cmdPptxOutline,
cmdPptxSlide,
cmdPptxText,
cmdSharepointResolve,
cmdTranscript,
cmdTranscriptOutline,
cmdVideoChapters,
cmdXlsxColumns,
cmdXlsxHead,
cmdXlsxQuery,
cmdXlsxRange,
cmdXlsxSheets,
} from './cli_office.js'
export {
cmdCsvProfile,
cmdCsvQuery,
cmdHtmlLint,
cmdHtmlOutline,
cmdHtmlQuery,
cmdJsonOutline,
cmdJsonQuery,
cmdOpenApiOp,
cmdOpenApiOutline,
cmdXmlOutline,
cmdXmlQuery,
cmdYamlOutline,
cmdYamlQuery,
cmdZipList,
cmdZipRead,
} from './cli_structured.js'
export {
cmdSqliteTables,
cmdSqliteSchema,
cmdSqliteQuery,
cmdImageMeta,
cmdImageText,
fenceOcrText,
} from './cli_cmd_formats.js'
export {
cmdSessionSchema,
cmdDescribe,
} from './session_store_schema.js'
function cmdPrSlice(pr: string, slice: string, opts: { repo?: string; json?: boolean }) {
process.exitCode = runPrSlice({ pr, slice, ...opts })
}
// Sets process.exitCode to the wrapped command's exit code (NOT via `guard`, which forces 0 on success — compress must propagate the real code so shell chaining still sees the original failure/success signal).
async function cmdCompress(
commandArgs: string[] | undefined,
opts: {
cmd?: string
cmdB64?: string
filter?: string
timeout?: string
compress?: boolean
profile?: string
maxTokens?: string
capHintB64?: string
quietSuccess?: boolean
native?: boolean
shell?: string
stdin?: boolean
} = {},
): Promise<void> {
const usedBase64 = opts.cmdB64 !== undefined
let payloadDamaged = false
try {
let command: string | undefined
for (let depth = 0; depth < 5; depth++) {
if (opts.cmdB64 !== undefined) {
payloadDamaged ||= base64PayloadDamaged(opts.cmdB64)
command = Buffer.from(opts.cmdB64, 'base64').toString('utf8')
delete opts.cmdB64
} else {
command = opts.cmd
if (Array.isArray(commandArgs) && commandArgs.length > 0) {
command = command ? [command, ...commandArgs].join(' ') : commandArgs.join(' ')
}
}
if (!usedBase64 || !command || !/^\s*(?:token-goat|tg)\s+(?:compress|bash|run)(?:\s|$)/.test(command)) break
// Reuse the CLI grammar so aliases, positional commands and runner options survive unwrapping without starting another token-goat process.
const { shlexSplit } = await import('./tool_filters/helpers.js')
const nested = buildProgram()
applyExitOverride(nested)
let parsedArgs: string[] = []
let parsedOpts: Record<string, unknown> = {}
nested.commands.find((sub) => sub.name() === 'compress')!.action((args: string[], innerOpts: Record<string, unknown>) => {
parsedArgs = args
parsedOpts = innerOpts
})
delete opts.cmd
await nested.parseAsync(shlexSplit(command).slice(1), { from: 'user' })
commandArgs = parsedArgs
opts = { ...opts, ...parsedOpts } as typeof opts
}
if (!command || command.trim() === '') {
err(formatCommandError('either command arguments, -c/--cmd, or --cmd-b64 is required'))
process.exitCode = 1
return
}
const compressionProfile = opts.profile === undefined ? undefined : parseCompressionProfile(opts.profile)
const bashRunner = await import('./bash_runner.js')
if (opts.compress === false) {
// Commander maps `--no-compress` to `opts.compress === false`.
process.exitCode = await bashRunner.runRaw(
command,
parseTimeout(opts.timeout, bashRunner.DEFAULT_TIMEOUT_SECONDS),
opts.native,
opts.shell,
...(opts.stdin === true ? [true] : []),
)
return
}
const maxTokens = opts.maxTokens !== undefined ? requireNonNegativeInt('--max-tokens', opts.maxTokens) : 0
process.exitCode = await bashRunner.run(command, {
filterName: opts.filter,
timeout: parseTimeout(opts.timeout, bashRunner.DEFAULT_TIMEOUT_SECONDS),
maxTokens,
...(opts.capHintB64 !== undefined ? { capHint: Buffer.from(opts.capHintB64, 'base64').toString('utf8') } : {}),
...(compressionProfile !== undefined ? { compressionProfile } : {}),
...(opts.quietSuccess === true ? { quietSuccess: true } : {}),
...(opts.native === true ? { nativeShell: true } : {}),
...(opts.shell !== undefined ? { shellType: opts.shell } : {}),
...(opts.stdin === true ? { rawStdin: true } : {}),
})
} catch (e) {
const code = (e as { code?: string }).code
if (code === 'commander.helpDisplayed' || code === 'commander.version' || code === 'commander.help') {
process.exitCode = 0
return
}
err(formatCommandError(e))
process.exitCode = 1
} finally {
if (payloadDamaged && process.exitCode !== undefined && process.exitCode !== 0) {
err('[tg: note] Command failed when invoked via --cmd-b64, and the payload did not decode cleanly (characters outside base64, a cut-off length, or bytes that are not UTF-8). If an LLM generated this base64 string, tokenization bit-drift may have corrupted characters or file paths. Always pass plain shell commands directly.')
}
}
}
// Whether a --cmd-b64 payload carries the marks of a typed string gone wrong: characters outside base64 or a length that does not re-encode to itself, or bytes that are not UTF-8. A payload the hooks encoded never does, so a failure behind one is the command's own and gets no bit-drift note.
function base64PayloadDamaged(payload: string): boolean {
const canonical = (s: string): string => s.replace(/\s|=+$/g, '').replace(/-/g, '+').replace(/_/g, '/')
const bytes = Buffer.from(payload, 'base64')
if (canonical(bytes.toString('base64')) !== canonical(payload)) return true
try {
new TextDecoder('utf-8', { fatal: true }).decode(bytes)
return false
} catch {
return true
}
}
// setTimeout's delay is a signed 32-bit millisecond count and larger values fire immediately, so the largest safe whole-second value is floor(2147483647 / 1000).
const MAX_TIMEOUT_SECONDS = 2147483
/** Resolve the --timeout flag (seconds), rounded up to whole seconds and clamped to the setTimeout limit: 0/absent/invalid → the built-in default. parseFloat, not parseInt, because hooks_bash.ts renders a large configured number as '1e+21', which parseInt reads as 1. */
export function parseTimeout(raw: string | undefined, fallbackSeconds: number): number {
const sec = raw ? parseFloat(raw) : 0
return sec > 0 ? Math.min(Math.ceil(sec), MAX_TIMEOUT_SECONDS) : fallbackSeconds
}
export * from './cli_skills.js'
export * from './cli_file_ops.js'
async function cmdGdriveSections(fileId: string, opts: { heading?: string; fresh?: boolean }): Promise<void> {
// An organisation that does not use Google Drive can switch the integration off entirely, which refuses here before any file id is validated or any connection is opened, and also stops the installed agent guidance from naming the command at all.
if (!loadConfig().gdrive.enabled) {
throw new CliError('gdrive-sections is disabled by gdrive.enabled = false in this install')
}
const { fetchDoc, getDocSections, formatSections, getSectionContent } = await import('./gdrive.js')
const fetchOpts = { fresh: opts.fresh === true }
// Fetch the whole doc once up front (honoring --fresh) so its raw byte size is available as the "full source" side of the bytes-saved calculation below, mirroring cmdSessionOutline/ cmdSessionSlice's convention. fetchDoc() always writes its result to the on-disk web-output cache before returning, so the getSectionContent/getDocSections calls below can safely pass `fresh: false` -- they read through to the entry this call just (re)populated, guaranteeing exactly one network fetch even with --fresh, instead of two.
const text = await fetchDoc(fileId, fetchOpts)
const fullSourceBytes = Buffer.byteLength(text, 'utf8')
let emitted: string
if (opts.heading !== undefined) {
const content = await getSectionContent(fileId, opts.heading, { fresh: false })
if (content === null) {
throw new CliError(`section ${quotedArg(opts.heading)} not found in document ${fileId}`)
}
emitted = `# ${opts.heading}\n${content}`
} else {
const sections = await getDocSections(fileId, { fresh: false })
emitted = formatSections(sections)
}
// A Google Doc is authorable by anyone who can edit the shared file, exactly like a fetched web page -- scan and fence it the same way `_applyFiltersAndPrint` does for WebFetch/web-output, under the same UNTRUSTED_WEB_TAG (this doc *is* fetched over HTTP, via performHttpFetch in gdrive.ts). Inlined rather than routed through `_applyFiltersAndPrint` because that helper's default head/tail elision would silently truncate output this command has always emitted in full; `emitted` is used unmodified below except when a match is found.
const toEmit = fenceUntrusted(emitted, UNTRUSTED_WEB_TAG)
out(toEmit)
// stats.ts's KIND_TO_SOURCE/COMMAND_KINDS registry had no `gdrive-sections`/`gdrive_sections` entry and nothing ever called recordStat for this command -- the dashboard bucket was permanently zero regardless of real usage, the same class of gap already fixed for map_lookup/changed_lookup/csv_query/brief_view/session_outline/session_slice (see project_runchanged_missing_stat memory).
const bytesSaved = cappedSourceBytesSaved(fullSourceBytes, Buffer.byteLength(toEmit, 'utf8'))
recordStat('gdrive_sections', bytesSaved, savedTokensFromBytes(bytesSaved))
}
// --- Program assembly -------------------------------------------------------
/** Commands that only query the index, so they may answer through a read-only connection when the index cannot be written (a read-only sandbox, a write-denied data directory): see db.ts's allowReadOnlyIndex. A command that writes the index as part of its job (index, worker, doctor, note, hook, mcp-serve, ...) must never be listed, because a writer given the read-only handle fails at its first write instead of at the open. */
const READ_ONLY_INDEX_COMMANDS: ReadonlySet<string> = new Set([
'symbol', 'read', 'brief', 'section', 'semantic', 'search', 'skeleton', 'outline', 'refs', 'answer', 'ask', 'map',
'exports', 'imports', 'find', 'locate', 'callers', 'call-chain', 'impact', 'dead', 'deps', 'types', 'scope',
'similar', 'context-for', 'test-for', 'importers', 'peek',
])
/** Commander's own `helpInformation` for the top-level program, captured before `buildProgram` shadows it with the compact grouped index. `help --full` calls this to emit the long per-command listing the compact index replaces; without it the long form is unreachable, since the override is an own property that hides the prototype method for every later caller. */
let originalHelpInformation: (() => string) | null = null
/** Build the Commander program. Exported so tests can introspect/parse it. */
export function buildProgram(): Command {
const program = new Command()
program
.name('token-goat')
.description('Surgical companion for AI coding agents that saves turns, context and tokens')
.version(VERSION, '-v, --version', 'print the token-goat version')
// Lets a caller (e.g. the VS Code extension) convey a project root explicitly instead of setting the spawned process's own working directory to it -- an attacker-controlled workspace should never be the cwd a shell/launcher resolves a binary name against, but commands that key off process.cwd() for project resolution still need a way to be told where that root is.
.option('--cwd <path>', 'run as if invoked from this directory (overrides the real working directory)')
// Lets a caller print a disclosure line ahead of the command's own output without composing two commands through a shell operator -- a rewritten command (see detectStructuralIndexRewrite in bash_structural_index.ts) needs to say what it substituted, and every shell parses one command with a global flag identically, where `echo ... &&` does not (no `&&` in PowerShell 5.1 at all).
.option('--notice <text>', 'print this line to stdout before the command\'s own output')
// Applied via a preAction hook (not inside `guard` below) so --cwd works for every command, not only the ones wrapped in `guard` -- the surgical-read commands (symbol, read, scope, ...) call runExit/runExitText directly and never go through guard, so a chdir living only inside guard silently no-ops for them. This hook fires before any command's action handler, guard-wrapped or not, and before anything resolves the project root or loads config.
program.hook('preAction', (thisCommand, actionCommand) => {
if (actionCommand.parent === program && READ_ONLY_INDEX_COMMANDS.has(actionCommand.name())) {
allowReadOnlyIndex(() => err('token-goat: the index database cannot be written here, so this run reads it without writing: no stats are recorded and changed files are not reindexed.'))
}
const opts = thisCommand.opts<{ cwd?: string; notice?: string }>()
const cwdOverride = opts.cwd
if (cwdOverride !== undefined) {
try {
process.chdir(cwdOverride)
} catch (e) {
throw new Error(`--cwd ${cwdOverride}: ${extractErrorMessage(e)}`, { cause: e })
}
}
if (opts.notice !== undefined) out(opts.notice)
})
// Each action wraps the (possibly sync) handler so any thrown CliError or unexpected error maps to a stderr line + exit code 1, and success to 0. A handler that already set process.exitCode itself (a deliberate non-zero exit without throwing) is left alone -- only the still-undefined default gets the success fallback.
const guard =
(fn: (...a: never[]) => void | Promise<void>) =>
async (...args: unknown[]): Promise<void> => {
process.exitCode = undefined
// loadConfig() silently falls back to defaults on a config.toml parse failure, same as when the file is simply missing -- surface the distinction here, once per invocation, so a corrupt config doesn't look identical to "no config yet" for every command.
loadConfig()
const parseErr = getLastConfigParseError()
if (parseErr !== null) {
// The parser quotes the offending line of the file back, so this banner carries file bytes in a line prefixed with token-goat's own name.
err(`token-goat: config.toml failed to parse (${displaySafeText(parseErr)}); using defaults — run \`token-goat config validate\` for details`)
}
// Same distinction as above, for the optional per-project .token-goat.toml override -- it fails open (global-only config still loads), but a corrupt project file should not look identical to "no project override" for every command.
const projectParseErr = getLastProjectConfigParseError()
if (projectParseErr !== null) {
// Same, and worse: a project override arrives with the repository, so these bytes are third-party on every clone. This banner prints before every command.
err(`token-goat: .token-goat.toml failed to parse (${displaySafeText(projectParseErr)}); ignoring project override`)
}
// A per-project file arrives with the repository, so it may not set the security controls an administrator configures once. Say which settings were ignored: silently dropping them would leave a legitimate author wondering why the file had no effect.
const lockedKeys = lastProjectConfigLockedKeys()
if (lockedKeys.length > 0) {
err(
`token-goat: .token-goat.toml may not set ${displaySafeText(lockedKeys.join(', '))}; ` +
'these are security settings and come from the global config or the environment only',
)
}
try {
await fn(...(args as never[]))
if (process.exitCode === undefined) {
process.exitCode = 0
}
} catch (e) {
err(formatCommandError(e))
process.exitCode = 1
}
}
registerInspectCommands(program, guard, { cmdSemantic, cmdSearch })
registerIndexCommands(
program,
guard,
{ cmdIndex, cmdMap, cmdBridgesStatus, cmdCommands, cmdMcpServe },
{
maxFilesScanned: MAX_FILES_SCANNED,
maxFilesScannedForced: MAX_FILES_SCANNED_FORCED,
inlineEmbedBudgetMs: INDEX_INLINE_EMBED_BUDGET_MS,
},
)
registerContentCommands(program, guard, {
cmdContentCompress,
cmdRetrieve,
cmdHandoffCreate,
cmdHandoffResolve,
runBenchCommand,
out,
})
registerIntegrationCommands(program, guard, {
cmdHook,
cmdInstall,
cmdUninstall,
cmdMcpStatus,
cmdWorkerStart,
cmdWorkerStop,
cmdWorkerStatus,
cmdHookServerRun,