Repository navigation
Expand file tree
/
Copy pathhooks_fetch.ts
More file actions
186 lines (162 loc) · 14 KB
/
Copy pathhooks_fetch.ts
File metadata and controls
186 lines (162 loc) · 14 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
import type { HookEvent } from './hook_registry.js';
import { registerHook } from './hook_registry.js';
import type { HookOutput } from './types.js';
import { passOutput, getToolName, getToolInput, denyOutput, extractToolResponseField, BODY_FIRST_TOOL_RESPONSE_KEYS, emitRewrite, emitRewriteIfChanged } from './hooks_common.js';
import { recordStat, savedTokensFromBytes } from './stats.js';
import { storeWebOutput, getWebOutput } from './web_cache.js';
import { recordWebFetch } from './session.js';
import { shortFingerprint } from './fingerprint.js';
import { loadConfig } from './config.js';
import { looksLikeHtml, extractCleanText } from './web_extract.js';
import { scanAndRecord, fenceWithMatches } from './untrusted_fence.js';
import { redactSecrets } from './secret_redact.js';
import { isRewriteWorthwhile, resolveMinNetSavingsBytes } from './tool_filters/base.js';
import { matchesAllowPattern, matchesDenyPattern, metadataEndpointRefusal } from './url_policy.js';
function extractToolResponse(raw: Record<string, unknown>): string {
return extractToolResponseField(raw, BODY_FIRST_TOOL_RESPONSE_KEYS);
}
/** Identity check shared by every WebFetch handler below: only a WebFetch call with a valid url is in scope. Deliberately does NOT require a session id -- see {@link resolveWebFetchContext} for the session-scoped variant used by the caching/dedup paths, and postFetchHandler's own comment for why the injection scan must run even without one. */
function resolveWebFetchUrl(event: HookEvent): { toolInput: Record<string, unknown>; url: string } | null {
const toolName = getToolName(event);
if (toolName !== 'WebFetch') {
return null;
}
const toolInput = getToolInput(event);
const url = toolInput['url'] as string;
if (!url || typeof url !== 'string') {
return null;
}
return { toolInput, url };
}
/** Shared prologue for the session-scoped caching/dedup paths (preFetchHandler's recall check, postFetchHandler's storeWebOutput): only a WebFetch call with a valid url AND a session id is in scope -- caching is inherently session-keyed, so a call with no session id has nothing to cache against. Everything else passes through untouched. */
function resolveWebFetchContext(event: HookEvent): { toolInput: Record<string, unknown>; url: string } | null {
if (!event.sessionId) {
return null;
}
return resolveWebFetchUrl(event);
}
/** Every URL a WebFetch call will reach: its `url`, then each string in a `urls` array. VS Code's fetch_webpage takes `urls[]` and names a single `url` only when it asked for one page (see VSCODE_DERIVED_INPUT in hooks_cli.ts), so a policy that read `url` alone let a call pass on its first page, or unjudged when it named several. */
function webFetchPolicyUrls(event: HookEvent): string[] {
if (getToolName(event) !== 'WebFetch') return [];
const toolInput = getToolInput(event);
const out: string[] = [];
const single = toolInput['url'];
if (typeof single === 'string' && single !== '') out.push(single);
const list = toolInput['urls'];
if (Array.isArray(list)) {
for (const entry of list) if (typeof entry === 'string' && entry !== '' && !out.includes(entry)) out.push(entry);
}
return out;
}
export function preFetchHandler(event: HookEvent): HookOutput {
try {
// webfetch.allow/webfetch.deny gate every WebFetch call regardless of session id -- unlike the dedup check below, blocking a URL has nothing to do with caching, so it must run even for a harness that sends no session_id (see resolveWebFetchContext's own comment). One refused URL refuses the whole call: the harness fetches the list as one tool call, so there is no way to let the rest through.
for (const url of webFetchPolicyUrls(event)) {
// Unconditional, and ahead of the configurable policy: an operator's allow/deny list is about which ordinary hosts they want reached, not about whether the machine's own cloud credentials are reachable through a fetched URL.
const metadataRefusal = metadataEndpointRefusal(url);
if (metadataRefusal !== null) {
return denyOutput(`WebFetch blocked: ${metadataRefusal}.`);
}
const wfCfg = loadConfig().webfetch;
if (wfCfg.deny.length > 0 && matchesDenyPattern(url, wfCfg.deny)) {
return denyOutput(`WebFetch blocked: URL matches a configured webfetch.deny pattern.`);
}
if (wfCfg.allow.length > 0 && !matchesAllowPattern(url, wfCfg.allow)) {
return denyOutput(`WebFetch blocked: URL does not match any configured webfetch.allow pattern.`);
}
}
const ctx = resolveWebFetchContext(event);
if (ctx === null) {
return passOutput();
}
const { toolInput, url } = ctx;
// Dedup key includes the prompt: WebFetch answers are prompt-specific, so a repeat fetch of the same URL with a different question must not be redirected to a stale answer for the WRONG question. Deriving the cache id directly (a pure function of url+prompt, mirroring cacheIdForUrl in web_cache.ts) instead of going through the session-scoped webFetches map also makes the lookup work across processes/sessions via getWebOutput's disk fallback - the same cross-process approach getWebOutputByUrlFromDisk uses for gdrive.ts.
const prompt = typeof toolInput['prompt'] === 'string' ? (toolInput['prompt'] as string) : '';
const cacheId = shortFingerprint(`${url}\x00${prompt}`);
// Guard on the content blob (in-memory or on disk), not just the session index, so a pruned or evicted entry never yields a web-output hint that would error - mirrors the curl-GET recall guard in hooks_bash.
const cached = getWebOutput(cacheId);
if (cached !== null) {
const cachedBytes = Buffer.byteLength(cached, 'utf-8');
if (cachedBytes >= loadConfig().hints.web_dedup_min_bytes) {
recordStat('webfetch:recall', cachedBytes, savedTokensFromBytes(cachedBytes));
return denyOutput(
'Already fetched this URL with this prompt; the response is cached. ' +
'Use `token-goat web-output ' + cacheId + ' --full` to recall it ' +
'(append `--grep PATTERN` to filter or `--section Heading` for a markdown section) instead of re-fetching.',
);
}
}
return passOutput();
} catch {
return passOutput();
}
}
export function postFetchHandler(event: HookEvent): HookOutput {
try {
// Deliberately resolved WITHOUT requiring a session id (unlike the caching path below): the injection scan's contract per README ("every fetched page is scanned for attack patterns") is unconditional, but relay.ts's sessionId derivation falls back to '' for any harness that doesn't send session_id/sessionId on the wire (see its own comment) -- a call from such a harness must still get scanned, even though it has nothing to cache against.
const urlCtx = resolveWebFetchUrl(event);
if (urlCtx === null) {
return passOutput();
}
const { toolInput, url } = urlCtx;
recordStat('web_fetch');
const body = extractToolResponse(event.raw);
// Prompt-injection scan: README's documented contract ("every fetched page is scanned for attack patterns") is unconditional, so this runs ahead of both the session gate and the caching-size gate below rather than being folded into either. A match anywhere in the response wraps the whole response in an untrusted-content fence before it reaches the model; the matched pattern names are written to the stats ledger's `detail` column. `injection.enabled` is the documented one-line opt-out.
const injectionMatches = body ? scanAndRecord(body) : [];
// Redact secrets on this same live path, computed here ahead of every early-return guard below -- neither WebFetch's caching store (storeWebOutput redacts its own persisted copy separately) nor this handler redacted what the model actually reads THIS turn, so a fetched page carrying a credential that trips no injection pattern (an API key pasted into a forum answer, a leaked token in an indexed gist) reached the model unredacted. Mirrors hooks_websearch.ts's postWebSearchHandler fix for the same gap.
const bodyRedacted = body ? redactSecrets(body) : { text: body, count: 0 };
// Everything below this point (dedup cache lookup key reuse aside, the actual store) is inherently session-scoped -- a missing session id has nothing to cache against, but the fence and redaction above must still apply to whatever was scanned.
if (!event.sessionId) {
// Unconditional: a fetched page is third-party web content by provenance, so it is fenced whether or not the eight deliberately-narrow patterns matched. It used to pass through unfenced on a clean scan, which meant any payload those patterns miss reached the model unmarked -- the fence's whole job is to survive a miss.
return emitRewriteIfChanged(body, fenceWithMatches(bodyRedacted.text, injectionMatches), 'fetch');
}
if (!body || body.length < 1024) {
// Unconditional: a fetched page is third-party web content by provenance, so it is fenced whether or not the eight deliberately-narrow patterns matched. It used to pass through unfenced on a clean scan, which meant any payload those patterns miss reached the model unmarked -- the fence's whole job is to survive a miss.
return emitRewriteIfChanged(body, fenceWithMatches(bodyRedacted.text, injectionMatches), 'fetch');
}
// Store under a (url, prompt) dedup key - see preFetchHandler - while keeping the persisted blob's displayed url clean for `token-goat web-history`. Also record it in the session (url-keyed) for the compact-manifest "fetched this session" listing.
const prompt = typeof toolInput['prompt'] === 'string' ? (toolInput['prompt'] as string) : '';
// A raw HTML body is the same unshrunk-payload problem image_shrink.ts already solves for images: extract clean text before caching, so a later recall (web-output, or a repeat fetch caught by preFetchHandler's dedup) never re-surfaces raw markup by default. webfetch.compress_bodies/compress_min_bytes already existed as persisted, validated, env-overridable config with zero consumers until now. The uncleaned body is passed through to storeWebOutput below as `rawContent` so it stays recoverable via `web-output --raw` rather than being lost -- a lossy store with no recovery path is the defect this closes, not the cleaning itself.
let storedBody = body;
const wfCfg = loadConfig().webfetch;
if (wfCfg.compress_bodies && body.length >= wfCfg.compress_min_bytes && looksLikeHtml(body)) {
try {
storedBody = extractCleanText(body);
} catch {
storedBody = body;
}
}
const cacheId = storeWebOutput(url, storedBody, `${url}\x00${prompt}`, storedBody !== body ? body : undefined);
recordWebFetch(url, prompt, cacheId);
// storedBody may differ from body (extractCleanText above), so redact it fresh rather than reusing bodyRedacted -- same "redact what is actually about to be shown" reasoning as fencing storedBody rather than the raw body just below.
const storedRedacted = storedBody === body ? bodyRedacted : redactSecrets(storedBody);
// Fence storedBody (the compressed copy just cached above), not the raw body -- fencing the raw body here would both defeat compress_bodies' token savings and return content that disagrees with what a later `token-goat web-output <id>` recall of the same cache entry would return. Unconditional: a fetched page is third-party web content by provenance, so it is fenced whether or not the eight deliberately-narrow patterns matched. It used to be fenced only on a hit, which meant any payload those patterns miss reached the model unmarked -- the fence's whole job is to survive a miss. The scan result now only decides whether the notice names pattern(s).
const fencedStored = fenceWithMatches(storedRedacted.text, injectionMatches);
// Ship the compressed copy already computed and cached above instead of discarding it -- previously storedBody was only ever consumed by the injection-detected branch, so a compressed HTML body's savings never reached the model. Gated on compression having actually happened (storedBody !== body) so an unchanged body is never re-credited, and on the same net-benefit floor bash_compress uses so a rewrite whose savings don't clear the recall notice's own cost isn't credited as compression.
if (storedBody !== body) {
const noticeFor = (id: string): string => `\n[token-goat: WebFetch body compressed via extractCleanText; use \`token-goat web-output ${id} --raw\` to recall it]`
const notice = noticeFor(cacheId)
const noticeBytes = Buffer.byteLength(notice, 'utf-8')
const originalBytes = Buffer.byteLength(body, 'utf-8');
// Measure the fenced string, which is what actually ships: gating on the unfenced length while emitting the fenced one is the same "gate(x), emit(x+extra)" over-report this branch's notice accounting was already fixed for once.
const rewrittenBytes = Buffer.byteLength(fencedStored, 'utf-8');
if (
isRewriteWorthwhile({
originalBytes,
rewrittenBytes,
noticeBytes,
minNetSavingsBytes: resolveMinNetSavingsBytes(),
})
) {
// Credit the delta against what is actually emitted (fenced body + recall notice), not against `rewrittenBytes`. The gate above already prices the notice in, but a hand-written recordStat beside the emit did not, so every compressed fetch over-reported its saving by the notice's own length -- and by any growth the redaction placeholders added on top. `emitRewrite`'s savings parameter measures the emitted string itself, which is the only figure that describes what the model was spared.
return emitRewrite(fencedStored + notice, 'fetch', { kind: 'webfetch:compress', originalBytes });
}
}
// Reached when the compression rewrite did not fire or did not pay off. The fence is a security action, not a compression one, so it must not inherit the net-benefit floor or the compress_bodies gate above.
return emitRewriteIfChanged(body, fencedStored, 'fetch');
} catch {
return passOutput();
}
}
registerHook('pre_tool_use', preFetchHandler, { toolName: 'WebFetch' });
registerHook('post_tool_use', postFetchHandler, { toolName: 'WebFetch' });