Repository navigation
Expand file tree
/
Copy pathhooks_mcp.ts
More file actions
211 lines (198 loc) · 16.7 KB
/
Copy pathhooks_mcp.ts
File metadata and controls
211 lines (198 loc) · 16.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
/** MCP read-tool caching hooks. post_tool_use: persist a read-only `mcp__*` result into the shared bash-output store. pre_tool_use: when an identical read-only `mcp__*` call was already cached this session, deny it and point at `token-goat bash-output <id>` so the model recalls a slice instead of paying the round trip again. Both handlers register with no toolName filter (MCP tool names are dynamic) and self-gate on {@link isMcpReadOnly} plus a present sessionId, so they are inert for every non-MCP and mutating tool. */
import { registerHook, type HookEvent } from './hook_registry.js'
import type { HookOutput } from './types.js'
import { getToolName, getToolInput, passOutput, denyOutput, extractToolResultText, isMcpErrorResponse, emitRewrite, emitRewriteIfChanged, mcpContentBlocks, isTextualContentBlock, withPreservedBlocks } from './hooks_common.js'
import { isMcpReadOnly, getMcpOutput, mcpOutputBytes, storeMcpOutput } from './mcp_cache.js'
import { PER_FILE_COUNTERFACTUAL_CEILING } from './util.js'
import { loadConfig } from './config.js'
import { compressMcpResult, MCP_COMPRESS_MIN_BYTES } from './mcp_compress.js'
import { compressMcpResultWithPacks } from './mcp_compress_packs.js'
import { redactSecrets } from './secret_redact.js'
import { UNTRUSTED_TOOL_TAG } from './injection_scan.js'
import { scanAndRecord, fenceWithMatches } from './untrusted_fence.js'
import { recordStat, savedTokensFromBytes } from './stats.js'
import { isRewriteWorthwhile, resolveMinNetSavingsBytes } from './tool_filters/index.js'
import { clipToDeliveryCap } from './delivery_cap.js'
import { MCP_TOOL_PATTERN } from './mcp_tool_pattern.js'
import { hintTarget } from './hint_target.js'
import { quotedArg } from './hint_suggestion_guard.js'
export const MCP_OVERSIZED_THRESHOLD_BYTES = 25_000
const TOKEN_GOAT_MCP_TOOL_NAMES = new Set([
'read',
'symbol',
'section',
'outline',
'skeleton',
'refs',
'brief',
'map',
'changed',
'grep',
'imports',
'exports',
'semantic',
'index_status',
'handoff_create',
'handoff_apply',
'retrieve_text',
])
/** Returns true if the tool belongs to token-goat's own local MCP server. Token-goat is a local companion process indexing the user's workspace files, not an external third-party remote API. Its tool results must not be fenced as untrusted remote payloads, which triggers prompt-injection refusals in LLMs. */
export function isTokenGoatMcpTool(toolName: string): boolean {
const lower = toolName.toLowerCase()
let rest = lower
if (rest.startsWith('mcp__')) {
rest = rest.slice(5)
} else if (rest.startsWith('mcp_')) {
rest = rest.slice(4)
} else {
if (lower.startsWith('token_goat_') || lower.startsWith('token-goat-') || lower.startsWith('tg_')) {
const tool = lower.replace(/^(token_goat_|token-goat-|tg_)/, '')
return TOKEN_GOAT_MCP_TOOL_NAMES.has(tool)
}
return false
}
const sepIdx = rest.indexOf('__')
let server = ''
let tool = ''
if (sepIdx !== -1) {
server = rest.slice(0, sepIdx)
tool = rest.slice(sepIdx + 2)
} else {
const singleIdx = rest.indexOf('_')
if (singleIdx !== -1) {
server = rest.slice(0, singleIdx)
tool = rest.slice(singleIdx + 1)
}
}
const isTgServer =
server === 'token_goat' ||
server === 'token-goat' ||
server === 'tg' ||
server === 'token_goat_mcp' ||
server === 'token-goat-mcp'
return isTgServer && TOKEN_GOAT_MCP_TOOL_NAMES.has(tool)
}
// Defined in hooks_common.ts alongside extractToolResultText, whose output the rule is about, and re-exported here because this module was its only caller for a while.
export { isMcpErrorResponse } from './hooks_common.js'
function preMcpHandler(event: HookEvent): HookOutput {
const toolName = getToolName(event)
if (!toolName || !event.sessionId || isTokenGoatMcpTool(toolName)) return passOutput()
const toolInput = getToolInput(event)
if (!isMcpReadOnly(toolName, toolInput)) return passOutput()
const ttlMs = loadConfig().hints.mcp_dedup_ttl_secs * 1000
const id = getMcpOutput(event.sessionId, toolName, toolInput, ttlMs)
if (!id) return passOutput()
// The denied call never runs, so its result never reaches the model a second time. Credit that blocked re-arrival the way preSkillHandler credits a blocked skill re-load: the cached entry's stored (redacted) size, capped at PER_FILE_COUNTERFACTUAL_CEILING because the counterfactual being priced is a response that would itself have been truncated. The full raw result is the right counterfactual here specifically because postMcpHandler returns early on a cache hit (see its getMcpOutput guard), so the allowed second call would have shipped the result uncompressed. No sibling stat double-counts these bytes: mcp:compress only ever fires on a call that actually ran, which this deny prevents.
const denyCredit = Math.min(mcpOutputBytes(id), PER_FILE_COUNTERFACTUAL_CEILING)
recordStat('mcp:recall', denyCredit, savedTokensFromBytes(denyCredit))
return denyOutput(
'Identical read-only MCP call already cached this session. Use `token-goat bash-output ' +
id +
'` to recall the result (add `--grep PATTERN`, `--tail N`, or `--head N` to slice) instead of repeating the call.',
)
}
function postMcpHandler(event: HookEvent): HookOutput {
const toolName = getToolName(event)
if (!toolName || !toolName.startsWith('mcp__') || isTokenGoatMcpTool(toolName)) return passOutput()
const toolInput = getToolInput(event)
const resultText = extractToolResultText(event.raw)
if (!resultText) return passOutput()
// An MCP result is a remote server's output: the least trusted text in the pipeline, and the one surface a page-only injection scan never covered. Scanned before every early return below, because each of those returns was a way for the same hostile text to reach the model unmarked: an in-band error response carries text just as a success does, a harness that omits the session id still shows the model the result, and the read-only dedup return fires on the SECOND of two identical calls -- whose result a hostile or time-varying server is free to make different from the first. Caching is what those guards exist to gate; the fence is not.
const injectionMatches = scanAndRecord(resultText)
// Redact secrets on this same live path, computed once here ahead of every early-return guard below -- an MCP result carrying a bare credential (an API key in an env-dump tool's output, a PAT sitting in a commit message) trips no injection pattern at all, so gating redaction on injectionMatches (as fenced() alone used to) would leave every non-fenced return -- no session id, an in-band MCP error, a dedup cache hit, and the terminal "compression did not fire or did not pay off" return below -- shipping it unredacted. Mirrors postWebSearchHandler's and postFetchHandler's fix for the same gap.
const redactedResult = redactSecrets(resultText)
// Every block whose payload no string can stand in for. The rewrite below replaces the tool result with one fenced text block, which represents each textual block completely (see isTextualContentBlock) but would drop an image, an audio clip or a blob resource outright -- measured: a mixed text+image MCP result came back as a bare string with its 220 KB picture simply gone. Computed once here, beside the text every branch shares, so a branch added later inherits it rather than forgetting it.
const preserved = (mcpContentBlocks(event.raw) ?? []).filter((b) => !isTextualContentBlock(b))
const preserve = (output: HookOutput): HookOutput => withPreservedBlocks(output, preserved)
const fenced = (): HookOutput =>
preserve(
emitRewriteIfChanged(
resultText,
fenceWithMatches(redactedResult.text, injectionMatches, UNTRUSTED_TOOL_TAG),
'mcp',
),
)
// Unconditional: an MCP result is a remote server's output by provenance, so it is fenced whether or not the eight deliberately-narrow patterns matched. It used to pass through unfenced on a clean scan, which meant any payload those patterns miss reached the model unmarked -- the fence's whole job is to survive a miss. The scan result now only decides whether the notice names pattern(s).
const passOrFence = (): HookOutput => fenced()
if (!event.sessionId) return passOrFence()
// An in-band MCP error is a valid response, not a cacheable one - never let a transient or now-resolved failure block every later identical retry.
if (isMcpErrorResponse(event.raw)) return passOrFence()
const readOnly = isMcpReadOnly(toolName, toolInput)
// Caching/dedup remains strictly read-only-gated: a mutating or non-idempotent call (e.g. a browser-automation `take_snapshot`, which trips MUTATING_VERBS_RE's `snapshot` token even though its result is compressible) must never be served back to a later identical pre_tool_use call, since its output can legitimately differ call to call.
let id: string | null = null
if (readOnly) {
const ttlMs = loadConfig().hints.mcp_dedup_ttl_secs * 1000
// Idempotent: a re-fired post for an already-cached, still-fresh call writes nothing.
if (getMcpOutput(event.sessionId, toolName, toolInput, ttlMs)) return passOrFence()
id = storeMcpOutput(event.sessionId, toolName, toolInput, resultText)
}
// Deterministic structural compression (see mcp_compress.ts and mcp_compress_packs.ts) is gated on compressibility, not on cache-eligibility: it used to require id !== null (i.e. the call was already read-only-cached), which silently excluded every non-idempotent tool a compression pack was written for -- e.g. chrome-devtools-mcp's take_snapshot, which trips MUTATING_VERBS_RE's `snapshot` token, never reached mcp_compress_packs.ts's dedicated browser-snapshot pack in production even though that pack exists specifically for it. Size-gated so small results are never touched, and opt-out (not opt-in) via TOKEN_GOAT_MCP_COMPRESS=0 to match TOKEN_GOAT_BASH_COMPRESS's existing convention elsewhere in this codebase. Per-server packs run first (GitHub, browser-automation): a schema-aware pack strips known boilerplate before handing the result to the same generic table-ifying pass. When no pack matches or pays off, the generic pass runs on the untransformed text exactly as before the packs existed.
if (resultText.length >= MCP_COMPRESS_MIN_BYTES && process.env['TOKEN_GOAT_MCP_COMPRESS'] !== '0') {
// Compress the already-redacted text (redactedResult, computed above before every early return), not raw resultText. mcp_compress_packs.ts's truncateSnapshotLine (and any other per-field truncation inside a pack or the generic compressor) can cut a credential mid-value; if that truncation runs on raw text before redaction, the surviving fragment can fall under a pattern's minimum-length floor and ship raw on this live, model-visible rewrite -- the same mechanism as the confirmed mcpInputPreview/tool_filters bugs. Feeding compression the pre-redacted text means every truncation point inside it only ever cuts placeholder text.
const compressed = compressMcpResultWithPacks(toolName, redactedResult.text) ?? compressMcpResult(redactedResult.text)
if (compressed !== null) {
// A mutating/non-idempotent call was never cached above (readOnly is false), but the "full via mcp-output <id>" label still needs somewhere to resolve, so store it now purely for recall -- this never feeds preMcpHandler's dedup check, which independently re-gates on isMcpReadOnly.
if (id === null) id = storeMcpOutput(event.sessionId, toolName, toolInput, resultText)
if (id !== null) {
// Defense-in-depth re-pass, normally a no-op now that compression consumes already-redacted text above: neither mcp_compress.ts nor mcp_compress_packs.ts run their own redaction, so this still catches any redaction-shaped text a pack's own transform might introduce.
const redactedBody = redactSecrets(compressed).text
const notice = `[token-goat: compressed, full via mcp-output ${id}]\n`
// Net-benefit gate (shared with bash_runner's filter pipeline, see tool_filters/base.ts::isRewriteWorthwhile): a rewrite that barely beats the original after paying for its own notice destabilises bytes that could otherwise be served from the provider's cached prefix for no real gain -- below the floor, ship resultText untouched instead.
const originalBytes = Buffer.byteLength(resultText, 'utf-8')
const worthwhile = isRewriteWorthwhile({
originalBytes,
rewrittenBytes: Buffer.byteLength(redactedBody, 'utf-8'),
noticeBytes: Buffer.byteLength(notice, 'utf-8'),
minNetSavingsBytes: resolveMinNetSavingsBytes(),
})
if (worthwhile) {
// If the compressed output is still oversized for the harness, clip the preview to avoid tool spill to content.json
const bodyBytes = Buffer.byteLength(redactedBody, 'utf-8')
let finalBody = redactedBody
if (bodyBytes >= MCP_OVERSIZED_THRESHOLD_BYTES) {
const preview = clipToDeliveryCap(redactedBody, Buffer.byteLength(notice, 'utf-8') + 500)
finalBody = preview.text + (preview.clipped ? '\n\n...[preview truncated by token-goat; use mcp-output above to query full payload]...' : '')
}
// MCP compression shipped for releases without recording anything, so the whole mechanism was invisible in `token-goat stats` even though the `mcp:` prefix was already registered in KIND_TO_SOURCE. Credited through emitRewrite's savings parameter rather than a hand-written recordStat beside the emit, because that parameter measures the emitted string itself -- notice, fence and redaction placeholders included -- which is the only figure that describes what the model was actually spared. A recordStat here would have to re-derive that per branch and would drift the moment either branch changed, which is exactly how the WebFetch over-report happened. Fenced on the same unconditional provenance rule as passOrFence above: a compressed result is still the remote server's text, and a clean scan is not evidence it is safe.
return preserve(
emitRewrite(
`${notice}${fenceWithMatches(finalBody, injectionMatches, UNTRUSTED_TOOL_TAG)}`,
'mcp',
{ kind: 'mcp:compress', originalBytes },
),
)
}
}
}
}
// Reached when compression did not fire or did not pay off. Oversized MCP result recovery: when a raw result is >= MCP_OVERSIZED_THRESHOLD_BYTES, Claude Code's internal harness will spill it to disk (e.g. content.json) and recommend reading it with read_file, bypassing surgical read discipline. Intercept by storing the full raw payload in the cache and returning a recovery notice with exact mcp-output query commands plus a clipped preview.
const rawBytes = Buffer.byteLength(resultText, 'utf-8')
if (rawBytes >= MCP_OVERSIZED_THRESHOLD_BYTES && process.env['TOKEN_GOAT_MCP_COMPRESS'] !== '0') {
if (id === null) id = storeMcpOutput(event.sessionId, toolName, toolInput, resultText)
if (id !== null) {
// A heading the payload really carries, double-quoted as hint_target.ts vetted it; this rewrite never passes the relay's suggestion guard, so the vetting is the only check it gets.
const heading = hintTarget('', 'section', { content: redactedResult.text })
const notice =
`[token-goat: oversized MCP result (${rawBytes} bytes) cached as ${id}]\n` +
`The full payload was cached to prevent harness context spill. Slicing commands:\n` +
` token-goat mcp-output ${id} --json-query '<path>' (e.g. 'issues[*].key', 'values[*].id')\n` +
` token-goat mcp-output ${id} --section ${heading.real ? quotedArg(heading.name) : "'<heading>'"}\n` +
` token-goat mcp-output ${id} --grep '<regex>' --max-matches 20\n` +
` token-goat mcp-output ${id} --head 50\n\n`
const preview = clipToDeliveryCap(redactedResult.text, Buffer.byteLength(notice, 'utf-8') + 500)
const previewBody = preview.text + (preview.clipped ? '\n\n...[preview truncated by token-goat; use mcp-output above to query full payload]...' : '')
return preserve(
emitRewrite(
`${notice}${fenceWithMatches(previewBody, injectionMatches, UNTRUSTED_TOOL_TAG)}`,
'mcp',
{ kind: 'mcp:oversized', originalBytes: rawBytes },
),
)
}
}
// The fence is a security action, not a compression one, so it must not inherit the size floor, the opt-out env var, or the net-benefit gate above -- a short hostile result is exactly the case those would drop.
return passOrFence()
}
// Both handlers no-op unless the tool name starts with `mcp__` (preMcpHandler via isMcpReadOnly, postMcpHandler via its own startsWith check), so declaring that prefix lets installHooks narrow the settings.json matcher without losing them.
registerHook('pre_tool_use', preMcpHandler, { toolPattern: MCP_TOOL_PATTERN })
registerHook('post_tool_use', postMcpHandler, { toolPattern: MCP_TOOL_PATTERN })