Repository navigation
Expand file tree
/
Copy pathbash_runner.ts
More file actions
425 lines (392 loc) · 30.3 KB
/
Copy pathbash_runner.ts
File metadata and controls
425 lines (392 loc) · 30.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
// Subprocess wrapper invoked by `token-goat compress` to run user commands.
//
// The pre-bash hook rewrites a Bash tool call from pytest -v tests/ to token-goat compress --filter pytest --cmd 'pytest -v tests/' When the harness executes the rewritten command, control lands here via `run()`: it runs the original command through the system shell, captures stdout and stderr, applies the requested filter, prints the compressed view with a one-line savings marker, records the byte/token savings, and exits with the *original* exit code so shell chaining (`cmd && next`) still works.
//
// Running through a shell is deliberate — the original command may use pipes, redirects, globs, env expansion, or chaining the user wrote intentionally. The command string was generated by the agent/user, not an untrusted source, and the hook's `detectFromCommand` already rejected pipelines/subshells before wrapping, so the wrapper adds no further escaping. The shell is chosen by wrappedShell(): /bin/sh on POSIX, the harness's Git-Bash on Windows (not cmd.exe, which cannot run the bash the harness wrote) — see src/shell.ts.
import * as os from 'node:os'
import { getBashOutput, storeBashOutputSync } from './bash_output_cache.js'
import { loadConfig } from './config.js'
import { isCompressionProfile, parseCompressionProfile, type CompressionProfile } from './config_types.js'
import { MAX_CAPTURE_BYTES } from './constants.js'
import { deliveredOutputBytes } from './delivery_cap.js'
import { displaySafeText } from './paths.js'
import { adaptPowerShellCommand } from './powershell_compat.js'
import { powerShellCommandArgs, resolvePowerShell, wrappedShell } from './shell.js' // tg-powershell-output-unread: runs the user's own command, not a token-goat process query
import { spawnOwned } from './process_tree.js'
import { recordStat } from './stats.js'
import { PassthroughFilter } from './tool_filters/generic.js'
import {
type CompressedOutput,
type ToolFilter,
compressedTokensSaved,
capTokens,
deliverCompressed,
dispatchArgv,
elidedLinesPointer,
filterByName,
selectFilter,
shlexSplit,
TOOL_FILTERS,
} from './tool_filters/index.js'
import { echoedValue } from './hint_suggestion_guard.js'
/** Default wall-clock timeout for the wrapped subprocess, in seconds. */
export const DEFAULT_TIMEOUT_SECONDS = 600
/** Minimum bytes saved before a stat row is worth writing (skips noise rows). */
const MIN_RECORD_STAT_BYTES = 32
/** First heartbeat delay for a long, quiet compressed command; each subsequent one doubles (15s, 30s, 60s, 120s, ...) so a 10-minute quiet run adds a handful of lines, not dozens. Injectable via {@link RunOptions.heartbeatIntervalMs} for tests; the shipping path always leaves it at this default. */
export const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000
/** Options for {@link run}. */
export interface RunOptions {
filterName?: string | undefined
timeout?: number
cwd?: string | undefined
env?: NodeJS.ProcessEnv | undefined
compressionProfile?: CompressionProfile | undefined
maxTokens?: number
/** Printed after the recall line when the output was cut, by `maxTokens` or by the filter's own line and byte limits:the narrower command that answers the question, since a capped dump is exactly the case where running it again is the wrong next move. */
capHint?: string | undefined
writeStdout?: (s: string) => void
writeStderr?: (s: string) => void
quietSuccess?: boolean | undefined
nativeShell?: boolean | undefined
shellType?: 'bash' | 'pwsh' | 'powershell' | 'native' | string | undefined
heartbeatIntervalMs?: number
/** Hand the wrapped command the caller's standard input; without it the command gets none, on every path. */
rawStdin?: boolean | undefined
}
/** Accumulates chunks up to `cap` total bytes, then silently drops the rest and flags `overflowed` so the caller can kill the child and append a truncation marker -- the async-spawn equivalent of `spawnSync`'s `maxBuffer` + `ENOBUFS`. */
class CaptureSink {
private readonly chunks: Buffer[] = []
private bytes = 0
overflowed = false
constructor(private readonly cap: number) {}
push(chunk: Buffer): void {
if (this.overflowed) return
this.bytes += chunk.length
if (this.bytes > this.cap) {
this.overflowed = true
return
}
this.chunks.push(chunk)
}
text(): string {
return decode(Buffer.concat(this.chunks))
}
}
/** Drives a doubling heartbeat timer from `startTime`: fires `write(elapsedSeconds)` after `intervalMs`, `intervalMs * 2`, `intervalMs * 4`, ... `onOutput()` resets the schedule back to `intervalMs`, so a chatty command that keeps producing output never sees a heartbeat at all -- only a genuinely quiet stretch does. */
export function scheduleHeartbeat(startTime: number, intervalMs: number, write: (elapsedSeconds: number) => void): { onOutput: () => void; stop: () => void } {
let interval = intervalMs
let timer: ReturnType<typeof setTimeout> | undefined
let pending: ReturnType<typeof setImmediate> | undefined
// Bumped by every output and by stop(), so a beat decided before either one never writes.
let epoch = 0
const cancel = (): void => {
epoch++
if (timer !== undefined) clearTimeout(timer)
if (pending !== undefined) clearImmediate(pending)
}
const arm = (): void => {
const armedAt = epoch
timer = setTimeout(() => {
// Timers run before the loop reads the pipes: after the loop was stalled the child's output is already waiting unread, so the beat is written from the check phase, after that read has reset the schedule (a stalled loop is not a quiet command).
pending = setImmediate(() => {
if (armedAt !== epoch) return
write(Math.round((Date.now() - startTime) / 1000))
interval *= 2
arm()
})
}, interval)
timer.unref()
}
arm()
return {
onOutput: (): void => {
interval = intervalMs
cancel()
arm()
},
stop: cancel,
}
}
/** Look up the filter by name first, falling back to argv-based dispatch, and return the argv that filter was chosen from alongside it. Selection and argv derivation share one `shlexSplit` + `dispatchArgv` here on purpose: they used to be computed independently (here and again in `wrapAndCompress`), so on a `cd DIR && grep ...` command the filter was picked from the peeled tokens while `compressOutput` still received an argv starting with `cd` -- every argv-reading filter then read the wrong flags/pattern/subcommand. */
export function resolveFilter(
command: string,
filterName: string | undefined,
cwd: string | undefined,
): { filter: ToolFilter | null; argv: string[]; unknownFilter?: string } {
// No caller-supplied cwd means the command runs in this process's own directory, which is the one whose package.json `npm test` / `yarn lint` resolves against. Leaving it undefined here left `resolvePackageManagerScript` unreachable from the CLI entirely, so a wrapped `compress -c "yarn lint"` fell through to the generic filter while the pre-hook, which does pass a cwd, picked the script's real one.
const effectiveCwd = cwd ?? process.cwd()
let split: string[] | null
try {
split = shlexSplit(command)
} catch {
split = null
}
// Same fallback wrapAndCompress used when the command is unsplittable: hand the filter the raw command as a single token.
const argv = split === null ? [command] : dispatchArgv(split, effectiveCwd).argv
if (filterName) {
const named = filterByName(filterName)
if (named !== null) return { filter: named, argv }
// Unknown name — fall through to auto-detect rather than failing, and report the name so run() can say so: a silent fall-through made `--filter az` look like a filter that compresses nothing.
const filter = split === null ? null : selectFilter(split, effectiveCwd)
return { filter, argv, unknownFilter: filterName }
}
if (split === null) return { filter: null, argv }
return { filter: selectFilter(split, effectiveCwd), argv }
}
// Resolve what spawnSync should actually invoke, plus any extra env needed to carry the command text intact. On Windows there is no real argv array at the OS level: CreateProcess always takes one command-line string, and Git-Bash's MSYS runtime reconstructs its own argv from that string with its own backslash-unescaping rules -- which fire the same way whether Node built the string via spawnSync's `shell` option or via a literal `['-c', command]` argv element, silently dropping a level of backslashes (`\\` becomes `\`) either way. Environment variables cross the process boundary as raw bytes with no command-line parsing at all, so carrying the command through `TG_CMD` and running `eval "$TG_CMD"` sidesteps the mangling entirely. Exported for tests, which spawn the PowerShell wrapper directly to compare its stdout and exit code with a direct `pwsh -Command` run.
export function spawnTarget(
command: string,
nativeShell: boolean | undefined,
shellType?: string | undefined,
): { file: string; args: string[]; shell: boolean; cmdEnv?: NodeJS.ProcessEnv } {
if (shellType === 'pwsh' || shellType === 'powershell') {
const ps = resolvePowerShell() // tg-powershell-output-unread: runs the user's own command, not a token-goat process query
// Exit mirrors `pwsh -Command`, which exits 1 when the final `$?` is false and 0 otherwise (`exit N` exits directly). Invoking a scriptblock always reports success and a try/finally would make statement-terminating errors fatal, so the wrapper parses the command and splices `if ($global:TgOk = $?) { }` (which captures `$?` without resetting it) at the end of the script and of each named begin, process, end and clean block (`$?` carries from one block into the next, so an empty block still reports its predecessor), before each top-level break/continue and at the end of each top-level finally; a top-level `return X` is rewritten to capture `$?` after X; a top-level break or continue keeps a label that some loop in the script declares, and that label gets a catching loop in case no enclosing loop has it, while any other label, quoted, dotted or computed at run time, is redirected to a catching `TgEsc` loop, so no label escapes the wrapper to turn its exit into 0, except a label that converts to an empty string (`""`, `$null`, `@()`), which pwsh treats as no label and so is left to stop the innermost loop, and since evaluating a computed label resets `$?`, its rewrite ends by re-failing `$?` through an ignored Write-Error when the capture before the break recorded a failure; a parse error is thrown as a ParserError quoting the failing line and exits 1, where direct pwsh exits 1 silently; and under Windows PowerShell 5.1, which never runs a process block under -Command, the script gets empty pipeline input so its process block is skipped there too. Known divergences: when the last output is formatted through a view with scriptblock columns (`return {1}`, a returned Process), direct pwsh can exit 0 after an earlier failure because formatting resets `$?`, where the wrapper exits 1 with identical output; and an error raised on a line holding a top-level return or break quotes that line with the capture spliced in.
return {
file: ps,
args: powerShellCommandArgs( // tg-powershell-output-unread: runs the user's own command, not a token-goat process query
'$TgC = $env:TG_CMD; Remove-Item Env:TG_CMD -ErrorAction SilentlyContinue; $global:TgOk = $true; $TgN = [Environment]::NewLine; $TgK = \'if ($global:TgOk = $?) { }\'; $TgT = $null; $TgE = $null; $TgA = [System.Management.Automation.Language.Parser]::ParseInput($TgC, [ref]$TgT, [ref]$TgE); $TgX = [System.Collections.Generic.List[object]]::new(); $TgL = [System.Collections.Generic.List[string]]::new(); $TgTop = { param($n) $p = $n.Parent; while ($p -isnot [System.Management.Automation.Language.ScriptBlockAst]) { $p = $p.Parent }; [object]::ReferenceEquals($p, $TgA) }; if ($TgE.Count -eq 0) { if ($TgA.EndBlock.Unnamed) { $TgX.Add([pscustomobject]@{ P = $TgC.Length; O = 0; L = 0; T = ($TgN + $TgK) }) }; foreach ($TgB in @($TgA.BeginBlock, $TgA.ProcessBlock, $TgA.EndBlock, $TgA.CleanBlock)) { if ($TgB -and -not $TgB.Unnamed) { $TgX.Add([pscustomobject]@{ P = ($TgB.Extent.EndOffset - 1); O = 0; L = 0; T = ($TgN + $TgK + $TgN) }) } }; foreach ($TgR in $TgA.FindAll({ param($n) $n -is [System.Management.Automation.Language.ReturnStatementAst] -and (& $TgTop $n) }, $true)) { if ($TgR.Pipeline -and $TgR.Pipeline.GetPureExpression()) { $TgX.Add([pscustomobject]@{ P = $TgR.Extent.StartOffset; O = 1; L = 6; T = \'$TgW = (\' }); $TgX.Add([pscustomobject]@{ P = $TgR.Extent.EndOffset; O = 1; L = 0; T = \'), $?; $global:TgOk = $TgW[1]; return $TgW[0]\' }) } else { $TgX.Add([pscustomobject]@{ P = $TgR.Extent.StartOffset; O = 1; L = 6; T = \'return $(\' }); $TgX.Add([pscustomobject]@{ P = $TgR.Extent.EndOffset; O = 1; L = 0; T = $(if ($TgR.Pipeline) { \'; \' + $TgK + \')\' } else { $TgK + \')\' }) }) } }; $TgD = @($TgA.FindAll({ param($n) $n -is [System.Management.Automation.Language.LabeledStatementAst] -and $n.Label }, $true) | ForEach-Object { $_.Label } | Select-Object -Unique); foreach ($TgR in $TgA.FindAll({ param($n) ($n -is [System.Management.Automation.Language.BreakStatementAst] -or $n -is [System.Management.Automation.Language.ContinueStatementAst]) -and (& $TgTop $n) }, $true)) { $TgX.Add([pscustomobject]@{ P = $TgR.Extent.StartOffset; O = 0; L = 0; T = ($TgK + \'; \') }); if ($TgR.Label -is [System.Management.Automation.Language.StringConstantExpressionAst]) { if ($TgR.Label.Value -eq \'\') { } elseif ($TgD -contains $TgR.Label.Value) { $TgL.Add($TgR.Label.Value) } else { $TgX.Add([pscustomobject]@{ P = $TgR.Label.Extent.StartOffset; O = 1; L = $TgR.Label.Extent.Text.Length; T = \'TgEsc\' }); $TgL.Add(\'TgEsc\') } } elseif ($TgR.Label) { $TgX.Add([pscustomobject]@{ P = $TgR.Label.Extent.StartOffset; O = 1; L = $TgR.Label.Extent.Text.Length; T = (\'$(if ((-not [string]($TgV = (\' + $TgR.Label.Extent.Text + \'))) -or (@(\' + (($TgD | ForEach-Object { "\'" + $_ + "\'" }) -join \',\') + \') -contains $TgV)) { $TgV } else { \'\'TgEsc\'\' }; if (-not $global:TgOk) { Microsoft.PowerShell.Utility\\Write-Error TgRestore -ErrorAction Ignore })\') }); foreach ($TgI in $TgD) { $TgL.Add($TgI) }; $TgL.Add(\'TgEsc\') } }; foreach ($TgR in $TgA.FindAll({ param($n) $n -is [System.Management.Automation.Language.TryStatementAst] -and $n.Finally -and $n.Finally.Statements.Count -gt 0 -and (& $TgTop $n) }, $true)) { $TgX.Add([pscustomobject]@{ P = ($TgR.Finally.Extent.EndOffset - 1); O = 0; L = 0; T = ($TgN + $TgK + $TgN) }) } } else { throw [System.Management.Automation.ParseException]::new($TgE) }; $TgS = $TgC; foreach ($TgI in ($TgX | Sort-Object -Property @{ Expression = \'P\'; Descending = $true }, @{ Expression = \'O\'; Descending = $false })) { $TgS = $TgS.Substring(0, $TgI.P) + $TgI.T + $TgS.Substring($TgI.P + $TgI.L) }; $TgO = $(if ($PSVersionTable.PSVersion.Major -lt 6) { \'do { @() | & ([scriptblock]::Create($TgS)) } while ($false)\' } else { \'do { & ([scriptblock]::Create($TgS)) } while ($false)\' }); foreach ($TgI in ($TgL | Select-Object -Unique)) { $TgO = \':\' + $TgI + \' do { \' + $TgO + \' } while ($false)\' }; . ([scriptblock]::Create($TgO)); if ($global:TgOk) { exit 0 } else { exit 1 }',
),
shell: false,
cmdEnv: { TG_CMD: adaptPowerShellCommand(command) },
}
}
const shell = nativeShell || shellType === 'native' ? true : wrappedShell()
if (typeof shell === 'string') return { file: shell, args: ['-c', 'eval "$TG_CMD"'], shell: false, cmdEnv: { TG_CMD: command } }
return { file: command, args: [], shell }
}
/** Merge the base env with any command-carrying env spawnTarget produced, without dropping the parent's environment when *env* itself is undefined (spawnSync's `env` option, once set, replaces rather than extends `process.env`). */
function mergeSpawnEnv(env: NodeJS.ProcessEnv | undefined, cmdEnv: NodeJS.ProcessEnv | undefined): NodeJS.ProcessEnv | undefined {
return cmdEnv ? { ...(env ?? process.env), ...cmdEnv } : env
}
/** Map a Node signal name to `128 + signum`, the shell exit convention. */
function signalExitCode(signal: NodeJS.Signals): number {
const num = (os.constants.signals as Record<string, number>)[signal]
return 128 + (num ?? 0)
}
/** Resolve the effective compression profile: explicit argument wins; otherwise read from config, treating `"auto"` or an unsupported persisted value as `"balanced"`. Config errors also fall back to `"balanced"`. */
function resolveProfile(explicit: CompressionProfile | undefined): CompressionProfile {
if (explicit !== undefined) return parseCompressionProfile(explicit)
try {
const p = loadConfig().compression.profile
return isCompressionProfile(p) ? p : 'balanced'
} catch {
return 'balanced'
}
}
/** Resolve the bash_compress line/byte compression caps from config, falling back to the tool-filter layer's own built-in defaults on load failure. */
function resolveCompressLimits(): { maxLines: number; maxBytes: number } {
try {
const bc = loadConfig().bash_compress
return { maxLines: bc.max_lines, maxBytes: bc.max_bytes }
} catch {
return { maxLines: 1000, maxBytes: 64 * 1024 }
}
}
/** The stderr line for a `--filter` name no filter carries: what ran instead, and the registered names with a word that starts with the one typed, so `--filter az` points at `azure-cli`. */
export function unknownFilterNotice(name: string, fallback: ToolFilter | null): string {
const typed = name.toLowerCase()
const names = ['generic', 'passthrough', ...new Set(TOOL_FILTERS.map((f) => f.name))]
// A word-start match, not a substring one: `az` is the start of `azure-cli` but sits inside `bazel` and `lazygit`, which are not what anyone typing it meant.
const near = names.filter((n) => n.split('-').some((word) => word.startsWith(typed)) || typed.startsWith(n)).slice(0, 5)
const instead = fallback === null ? 'running it uncompressed' : `auto-detected ${fallback.name}`
const hint = near.length > 0 ? `; did you mean ${near.join(', ')}?` : ''
return `[token-goat: no filter named ${echoedValue(name)}, ${instead}${hint}]\n`
}
/** Run *command* through the system shell, compress its output, and return the wrapped subprocess's exit code (124 on wrapper-induced timeout, `128 + signum` when killed by a signal). When no filter applies the command is streamed through unchanged so the cost is limited to one subprocess fork. */
export async function run(command: string, opts: RunOptions = {}): Promise<number> {
const timeout = opts.timeout ?? DEFAULT_TIMEOUT_SECONDS
const { filter, argv, unknownFilter } = resolveFilter(command, opts.filterName, opts.cwd)
if (unknownFilter !== undefined) (opts.writeStderr ?? ((s: string) => process.stderr.write(s)))(unknownFilterNotice(unknownFilter, filter))
if (filter === null) {
// No tool filter matches this command. Ordinarily that means streaming it through raw is cheapest (one subprocess fork, no capture). But when the caller asked for a `--max-tokens` cap or `--quiet-success`, raw passthrough would silently bypass it — `passthrough()` uses `stdio: 'inherit'` and never sees the output. Route through the capture-and-compress path with an identity filter instead, so the options apply.
if ((opts.maxTokens ?? 0) > 0 || opts.quietSuccess) {
return wrapAndCompress(command, argv, new PassthroughFilter(), timeout, resolveProfile(opts.compressionProfile), opts)
}
return passthrough(command, timeout, opts.cwd, opts.env, opts.nativeShell, opts.shellType, opts.rawStdin)
}
return wrapAndCompress(command, argv, filter, timeout, resolveProfile(opts.compressionProfile), opts)
}
/** The child's stdin on every run path: none, unless the caller asked for its own (`compress --stdin`), then `forward` (inherit for a streamed run, a pipe for a captured one). A hook-wrapped command must never wait on input the harness will not send, which is why 'ignore' is the default. */
function childStdin(rawStdin: boolean | undefined, forward: 'inherit' | 'pipe'): 'ignore' | 'inherit' | 'pipe' {
return rawStdin === true ? forward : 'ignore'
}
/** Run *command* raw with no compression, inheriting the parent's stdout and stderr and returning its exit code. Used by `compress --no-compress` to debug the wrapper by streaming output straight through. */
export function runRaw(
command: string,
timeout: number = DEFAULT_TIMEOUT_SECONDS,
nativeShell?: boolean,
shellType?: string,
rawStdin?: boolean,
): Promise<number> {
return passthrough(command, timeout, undefined, undefined, nativeShell, shellType, rawStdin)
}
/** Run *command* with no compression, inheriting the parent's stdout and stderr. The command runs as an owned process tree (see {@link spawnOwned}): the timeout kills the shell and everything it started, not just the shell. */
async function passthrough(
command: string,
timeout: number,
cwd: string | undefined,
env: NodeJS.ProcessEnv | undefined,
nativeShell?: boolean,
shellType?: string,
rawStdin?: boolean,
): Promise<number> {
const { file, args, shell, cmdEnv } = spawnTarget(command, nativeShell, shellType)
const owned = spawnOwned(file, args, { cwd, shell, env: mergeSpawnEnv(env, cmdEnv), stdio: [childStdin(rawStdin, 'inherit'), 'inherit', 'inherit'] })
let timedOut = false
const timeoutHandle = setTimeout(() => {
timedOut = true
owned.kill()
}, timeout * 1000)
// 'error' (spawn failure, e.g. ENOENT) and 'close' can both fire; the promise settles on the first.
const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null; error?: Error }>((resolve) => {
owned.child.once('error', (error) => resolve({ code: null, signal: null, error }))
owned.child.once('close', (code, signal) => resolve({ code, signal }))
})
clearTimeout(timeoutHandle)
owned.release()
if (timedOut) return 124
// A spawn failure (ENOENT) leaves no status; exit 127 like a shell's "command not found" instead of reporting success.
if (result.error) {
process.stderr.write(`[token-goat: ${displaySafeText(result.error.message)}]\n`)
return 127
}
if (result.code !== null) return result.code
if (result.signal) return signalExitCode(result.signal)
return 0
}
/** Decode captured bytes as UTF-8 with replacement; null/undefined → empty. */
function decode(buf: Buffer | string | null | undefined): string {
if (buf == null) return ''
return typeof buf === 'string' ? buf : buf.toString('utf8')
}
/** Run *command* with output capture, apply *filter*, print the compressed view. `argv` is the dispatch argv the filter was selected from (see {@link resolveFilter}), NOT re-derived here. */
async function wrapAndCompress(
command: string,
argv: string[],
filter: ToolFilter,
timeout: number,
profile: CompressionProfile,
opts: RunOptions,
): Promise<number> {
const writeStdout = opts.writeStdout ?? ((s: string) => process.stdout.write(s))
const writeStderr = opts.writeStderr ?? ((s: string) => process.stderr.write(s))
const startTime = Date.now()
const { file, args, shell, cmdEnv } = spawnTarget(command, opts.nativeShell, opts.shellType)
const stdoutSink = new CaptureSink(MAX_CAPTURE_BYTES)
const stderrSink = new CaptureSink(MAX_CAPTURE_BYTES)
let timedOut = false
let overflowed = false
// Async spawn (not spawnSync) is required for the heartbeat below: a blocking spawnSync call never lets the event loop tick, so no timer could ever fire while the child is still running -- this is the actual root cause of `token-goat compress` looking hung on a long, quiet command, not just a missing print statement.
const owned = spawnOwned(file, args, { cwd: opts.cwd, shell, env: mergeSpawnEnv(opts.env, cmdEnv), stdio: [childStdin(opts.rawStdin, 'pipe'), 'pipe', 'pipe'] })
const child = owned.child
if (child.stdin) {
child.stdin.on('error', () => {})
process.stdin.on('error', () => {})
process.stdin.pipe(child.stdin)
}
const heartbeat = scheduleHeartbeat(startTime, opts.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS, (elapsed) => {
writeStderr(`[token-goat compress] still running, ${elapsed}s elapsed\n`)
})
const timeoutHandle = setTimeout(() => {
timedOut = true
owned.kill()
// A descendant that was reparented away from the shell can hold the pipes open past the shell's death and 'close' would never fire: once the shell is gone (now, or when its exit event arrives after the kill) stop reading them so the run settles as a timeout.
const stopReading = (): void => {
child.stdout?.destroy()
child.stderr?.destroy()
}
if (child.exitCode !== null || child.signalCode !== null) stopReading()
else child.once('exit', stopReading)
}, timeout * 1000)
const onData = (sink: CaptureSink) => (chunk: Buffer): void => {
heartbeat.onOutput()
sink.push(chunk)
if (sink.overflowed) {
overflowed = true
owned.kill()
}
}
child.stdout?.on('data', onData(stdoutSink))
child.stderr?.on('data', onData(stderrSink))
// 'error' (spawn failure, e.g. ENOENT) and 'close' can both fire; the promise settles on the first, and a spawn error maps to 127 with its message kept for stderr.
let spawnError: Error | null = null
const { code, signal } = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => {
child.on('error', (e) => {
spawnError = e
resolve({ code: 127, signal: null })
})
child.on('close', (code, signal) => resolve({ code, signal }))
})
heartbeat.stop()
clearTimeout(timeoutHandle)
owned.release()
// Release the caller's stdin explicitly instead of relying on the pipe's own cleanup, so a library caller's event loop is never held open by it.
if (child.stdin) {
process.stdin.unpipe(child.stdin)
process.stdin.pause()
}
let stdoutText = stdoutSink.text()
let stderrText = stderrSink.text()
if (overflowed) {
stdoutText += `\n[token-goat: capture capped at ${MAX_CAPTURE_BYTES / (1024 * 1024)} MiB]`
}
if (spawnError !== null) stderrText = (stderrText ? stderrText + '\n' : '') + `[token-goat: ${displaySafeText((spawnError as Error).message)}]`
let exitCode: number
if (timedOut) {
exitCode = 124
stderrText = (stderrText ? stderrText + '\n' : '') + `[token-goat: command exceeded ${timeout}s timeout and was killed]`
} else if (overflowed && (signal === 'SIGKILL' || process.platform === 'win32')) {
// The overflow kill used to end the shell with SIGTERM (143); the tree kill is a SIGKILL on POSIX and a plain status 1 from taskkill on Windows, so the old code is kept.
exitCode = signalExitCode('SIGTERM')
} else if (code !== null) {
exitCode = code
} else if (signal) {
exitCode = signalExitCode(signal)
} else {
exitCode = 0
}
if (opts.quietSuccess && exitCode === 0) {
const duration = ((Date.now() - startTime) / 1000).toFixed(1) + 's'
const fullRaw = (stdoutText + (stderrText ? '\n' + stderrText : '')).trim()
const id = storeBashOutputSync(command, fullRaw, 0, opts.cwd ?? null)
const summary = `[tg: ok] ${command} (${duration}, recall: token-goat bash-output ${id})\n`
writeStdout(summary)
return 0
}
const limits = resolveCompressLimits()
// The net-benefit gate lives in `deliverCompressed`: a rewrite whose bytesSaved doesn't clear the marker's own cost plus the configured floor destabilises the bytes (breaks provider prefix caching) for a saving too small to be worth it, so below the floor that function returns the ORIGINAL streams with no marker -- exactly as though no filter had matched. `token-goat bench` measures delivery through the same function, so a benchmark cannot drift from what actually ships.
const { applied, marker, compressed, text: delivered } = deliverCompressed(filter, stdoutText, stderrText, exitCode, argv, {
compressionProfile: profile,
maxLines: limits.maxLines,
maxBytes: limits.maxBytes,
})
// Apply the pressure-scaled token cap to the body BEFORE appending the marker so the savings marker survives truncation and stays visible to the agent.
let text = delivered
const maxTokens = opts.maxTokens ?? 0
if (maxTokens > 0) text = capTokens(text, maxTokens)
const capped = text !== delivered
let body = text + marker
// A cap that cut the output is a loss the same as a filter that did, so it gets the same recall. Keyed on `applied` alone, a passthrough run capped to 2,000 tokens dropped the rest with nothing to recover it from.
if (applied || capped) {
// The marker's own notice names TOKEN_GOAT_BASH_COMPRESS as the way to disable compression, and a rerun carrying it as a prefix is left unwrapped by the pre-hook, but that costs the command a second run. A recall of the untruncated bytes is the cheaper follow-up, so store them the same way the compound post-hook path already does and point at it.
const fullRaw = (stdoutText + (stderrText ? '\n' + stderrText : '')).trim()
const recallId = storeBashOutputSync(command, fullRaw, exitCode, opts.cwd ?? null)
body += '\n[token-goat] full output: token-goat bash-output ' + recallId + ' --full'
// A cut that removed one run of lines is recalled as exactly that run, checked against the copy bash-output reads, so the follow-up costs those lines and not the whole output again.
const stored = getBashOutput(recallId)
if (stored !== null) body += elidedLinesPointer(recallId, () => stored.output, text)
if (opts.capHint) body +='\n[token-goat] ' + opts.capHint
}
writeStdout(body.endsWith('\n') ? body : body + '\n')
if (applied) recordSavings(compressed)
return exitCode
}
/** Best-effort: record the savings stat. recordStat already swallows DB errors. Measured against the DELIVERED size, not the original. The harness caps how much of a Bash result reaches the model (see {@link deliveredOutputBytes}), so compressing a 30 MB output to 8 KB spares it at most `cap - 8 KB`, never 30 MB. Booking `result.bytesSaved` credited output the model was never going to be shown. The APPLY decision above deliberately still runs on the uncapped bytes. The two answer different questions: the gate asks whether a compressed body beats the harness's arbitrary head-truncation of the same output (it does -- the model sees a coherent summary instead of the first 20 KB), while this figure asks how many tokens were actually spared. Capping the gate would suppress the rewrites with the most to offer, which is a behaviour change this accounting fix must not make. */
function recordSavings(result: CompressedOutput): void {
const delivered = deliveredOutputBytes(result.originalBytes)
const bytesSaved = Math.max(0, delivered - result.compressedBytes)
if (bytesSaved < MIN_RECORD_STAT_BYTES) return
recordStat(`bash_compress:${result.filterName}`, bytesSaved, compressedTokensSaved(bytesSaved))
}