Repository navigation
Expand file tree
/
Copy pathwebfetch.ts
More file actions
369 lines (329 loc) · 17.1 KB
/
Copy pathwebfetch.ts
File metadata and controls
369 lines (329 loc) · 17.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
import { existsSync, readdirSync, statSync, unlinkSync } from 'fs';
import type * as HttpModule from 'http';
import * as https from 'https';
import { createRequire } from 'module';
import { isIPv4, isIPv6 } from 'net';
import { resolve, join } from 'path';
import { URL } from 'url';
import { promisify } from 'util';
import { lookup as dnsLookup, type LookupOptions } from 'dns';
import { dataDir } from './constants.js';
import { loadConfig } from './config.js';
import { isPrivateIpv4Octets, isPrivateIpv6Groups, urlPolicyDenialReason } from './url_policy.js';
// Loaded through require, never `import`: on Node 22 an ESM import of `http` reads every export to build its facade, one of them is a lazy WebSocket getter that loads undici, and undici compiles WebAssembly as it loads, so under --jitless every command died at startup (tests/guards/bundle_never_imports_http_as_esm.test.ts).
const http = createRequire(import.meta.url)('http') as typeof HttpModule;
const dnsLookupAsync = promisify(dnsLookup);
const MAX_URL_IN_ERROR = 200;
const ALLOW_UNRESOLVED = (process.env['TOKEN_GOAT_WEBFETCH_ALLOW_UNRESOLVED'] ?? '').toLowerCase() === 'true';
function webCacheDir(): string {
return join(dataDir(), 'web_cache');
}
const BLOCKED_HOSTNAMES = new Set([
'localhost',
'ip6-localhost',
'ip6-loopback',
'metadata.google.internal',
'169.254.169.254',
]);
function truncateUrl(url: string, maxLen: number = MAX_URL_IN_ERROR): string {
const sanitized = url.replace(/[\r\n]/g, '');
return sanitized.length > maxLen ? sanitized.slice(0, maxLen) + '…' : sanitized;
}
type SsrfResolution =
| { kind: 'safe'; address: string; family: number; addresses: Array<{ address: string; family: number }> }
| { kind: 'blocked' }
| { kind: 'unresolved' };
/** Resolve `hostname` and validate every returned address is not private — the single source of truth for "is this host safe to connect to". Used as the resolver plugged into every real socket connection (see ssrfPinnedLookup below), so the address that gets validated is always the exact address that gets connected to — closing the DNS-rebinding TOCTOU gap between a separate check and fetch. */
async function resolveSsrfSafeAddress(hostname: string): Promise<SsrfResolution> {
const hostnameLower = hostname.toLowerCase().replace(/\.$/, '');
if (BLOCKED_HOSTNAMES.has(hostnameLower)) return { kind: 'blocked' };
let results: Array<{ address: string; family: number }>;
try {
const r = await dnsLookupAsync(hostnameLower, { all: true });
if (!Array.isArray(r) || r.length === 0) return { kind: 'unresolved' };
results = r;
} catch {
return { kind: 'unresolved' };
}
for (const addr of results) {
const ip = addr.address || '';
const isPrivate = isIPv4(ip) ? isPrivateIPv4(ip) : isIPv6(ip) ? isPrivateIPv6(ip) : true;
if (isPrivate) return { kind: 'blocked' };
}
const first = results[0];
if (!first) return { kind: 'unresolved' };
return { kind: 'safe', address: first.address, family: first.family, addresses: results };
}
/** dns.lookup-compatible resolver passed as the `lookup` option on every real HTTP(S) request this module makes (initial request and every redirect hop). Resolving here — rather than trusting a hostname string handed to http(s).request — means the address used to open the TCP/TLS socket is the exact address resolveSsrfSafeAddress just validated, with no gap for a second, independent DNS lookup (and thus DNS rebinding) to slip in. Node's own dual-stack (Happy Eyeballs / autoSelectFamily, default-on since Node 20) connection logic sets `options.all` when it wants every resolved address back to race connections across, and requires the array-callback shape (`callback(err, addresses[])`) in that case -- not the single `callback(err, address, family)` shape used otherwise. Always returning the single-address shape regardless of `options.all` made Node's net internals throw "Invalid IP address: undefined" for any real, non-literal hostname (verified against raw.githubusercontent.com fetch-image dogfood run on Node 24) -- every fetch of a real URL was broken, not just SSRF targets. Both branches below now answer in whichever shape `options.all` asked for. */
export function ssrfPinnedLookup(
hostname: string,
options: LookupOptions,
callback: (
err: NodeJS.ErrnoException | null,
address: string | Array<{ address: string; family: number }>,
family?: number,
) => void,
): void {
resolveSsrfSafeAddress(hostname)
.then((result) => {
if (result.kind === 'safe') {
if (options.all) {
callback(null, result.addresses);
} else {
callback(null, result.address, result.family);
}
return;
}
if (result.kind === 'unresolved' && ALLOW_UNRESOLVED) {
// Even though the pinned (all:true) lookup didn't resolve, whatever address this raw fallback lookup DOES come back with must still be validated before it's handed to the socket — otherwise a hostname engineered to fail the pinned lookup but resolve here (a DNS rebinding technique) would bypass private-IP blocking entirely.
dnsLookup(hostname, { ...options, all: false }, (err, address, family) => {
if (err) {
callback(err, options.all ? [] : '', family);
return;
}
const addr = typeof address === 'string' ? address : '';
const isPrivate = isIPv4(addr) ? isPrivateIPv4(addr) : isIPv6(addr) ? isPrivateIPv6(addr) : true;
if (isPrivate) {
callback(
new Error(`URL blocked by SSRF safety check: ${truncateUrl(hostname)}`) as NodeJS.ErrnoException,
options.all ? [] : '',
);
return;
}
if (options.all) {
callback(null, [{ address: addr, family: family ?? 0 }]);
} else {
callback(null, addr, family);
}
});
return;
}
callback(
new Error(`URL blocked by SSRF safety check: ${truncateUrl(hostname)}`) as NodeJS.ErrnoException,
options.all ? [] : '',
);
})
.catch((err: unknown) => {
callback(err instanceof Error ? (err as NodeJS.ErrnoException) : new Error(String(err)), options.all ? [] : '');
});
}
export function isPrivateIPv4(ip: string): boolean {
const octets = ip.split('.').map((s) => {
const n = Number(s);
return s.length > 0 && Number.isFinite(n) && n >= 0 && n <= 255 ? n : NaN;
});
if (octets.length !== 4 || octets.some((o) => !Number.isFinite(o))) return false;
// Range table lives in url_policy.ts's isPrivateIpv4Octets, shared with screenshot.ts's headless-browser navigation policy so the two channels judge the same address the same way instead of drifting into two independently maintained range tables.
return isPrivateIpv4Octets(octets[0] as number, octets[1] as number, octets[2] as number);
}
/** Parse a syntactically valid IPv6 literal (per net.isIPv6) into its 8 16-bit groups, expanding "::" zero-compression and any embedded dotted-decimal IPv4 suffix (the IPv4-mapped/IPv4-translated forms, e.g. `::ffff:1.2.3.4` or `::ffff:0:1.2.3.4`). Returns null only if `ip` fails net.isIPv6 or otherwise can't be decoded. */
function parseIPv6Groups(ip: string): number[] | null {
if (!isIPv6(ip)) return null;
const withoutZone = ip.split('%')[0] ?? ip; // strip a zone/scope id, e.g. "fe80::1%eth0"
const halves = withoutZone.split('::');
if (halves.length > 2) return null;
const parseSide = (side: string): number[] | null => {
if (side === '') return [];
const tokens = side.split(':');
const groups: number[] = [];
for (let i = 0; i < tokens.length; i++) {
const tok = tokens[i] as string;
if (tok.includes('.')) {
if (i !== tokens.length - 1 || !isIPv4(tok)) return null; // embedded IPv4 must be the final token
const octets = tok.split('.').map(Number);
groups.push(((octets[0] as number) << 8) | (octets[1] as number));
groups.push(((octets[2] as number) << 8) | (octets[3] as number));
} else {
const val = parseInt(tok, 16);
if (!Number.isFinite(val) || val < 0 || val > 0xffff) return null;
groups.push(val);
}
}
return groups;
};
if (halves.length === 1) {
const groups = parseSide(halves[0] ?? '');
return groups && groups.length === 8 ? groups : null;
}
const left = parseSide(halves[0] ?? '');
const right = parseSide(halves[1] ?? '');
if (!left || !right) return null;
const missing = 8 - left.length - right.length;
if (missing < 0) return null;
return [...left, ...new Array(missing).fill(0), ...right];
}
/** True when `ip` is an IPv6 loopback/unique-local/link-local address, or an IPv4-mapped (`::ffff:a.b.c.d`), IPv4-translated (`::ffff:0:a.b.c.d`), or the deprecated IPv4-compatible (`::a.b.c.d`) address whose embedded IPv4 address is private. The actual range table (including the 6to4/NAT64 transition prefixes and fc00::/7, fe80::/10, fec0::/10, ff00::/8) lives in url_policy.ts's isPrivateIpv6Groups, shared with screenshot.ts's headless-browser navigation policy so the two channels can't drift. */
export function isPrivateIPv6(ip: string): boolean {
const lower = ip.toLowerCase();
if (lower === '::1') return true; // loopback
const groups = parseIPv6Groups(lower);
if (!groups) return true; // couldn't decode a claimed-valid IPv6 address — fail closed
return isPrivateIpv6Groups(groups);
}
// A download in progress writes its .tmp file continuously; 10 minutes with no further writes is long enough that any real fetch (even a slow one) would have finished or timed out on its own, so anything older is safe to treat as abandoned. Below this age, the file is left alone -- deleting it out from under an active concurrent download would corrupt or lose that fetch.
const STALE_DOWNLOAD_AGE_MS = 10 * 60 * 1000;
export function cleanupStaleDownloads(): number {
const cacheDir = webCacheDir();
if (!existsSync(cacheDir)) return 0;
let removed = 0;
try {
const files = readdirSync(cacheDir);
for (const file of files) {
if (file.endsWith('.tmp')) {
const filePath = resolve(cacheDir, file);
try {
const stat = statSync(filePath);
if (Date.now() - stat.mtimeMs < STALE_DOWNLOAD_AGE_MS) continue;
unlinkSync(filePath);
removed++;
} catch {
// Ignore stat/removal errors (e.g. file removed concurrently by its own download)
}
}
}
} catch {
// Ignore directory read errors
}
return removed;
}
export interface HttpFetchResult {
status: number;
statusText: string;
headers: Record<string, string>;
body: Buffer;
}
export interface HttpFetchOpts {
deadlineAt: number;
timeoutSec: number;
maxSizeBytes: number;
requestHeaders: Record<string, string>;
redirectsLeft: number;
}
export function performHttpFetch(targetUrl: string, opts: HttpFetchOpts): Promise<HttpFetchResult> {
return new Promise((resolvePromise, rejectPromise) => {
let parsed: URL;
try {
parsed = new URL(targetUrl);
} catch {
rejectPromise(new Error(`Invalid URL: ${truncateUrl(targetUrl)}`));
return;
}
if (!['http:', 'https:'].includes(parsed.protocol)) {
rejectPromise(new Error(`URL blocked by SSRF safety check: ${truncateUrl(targetUrl)}`));
return;
}
// Offline mode is checked here rather than at each command, because this is the one function every fetch token-goat performs itself goes through, redirects included.
if (loadConfig().network.offline) {
rejectPromise(new Error(`Offline mode is on (network.offline): refusing to fetch ${truncateUrl(targetUrl)}`));
return;
}
// webfetch.allow/webfetch.deny is the operator's egress policy, and it used to be enforced in exactly one place: the WebFetch pre-hook, which gates the harness's fetch tool. Every fetch token-goat performs itself -- `fetch-image`, `gdrive-sections` -- comes through here instead and ignored the policy completely, so an install configured to deny everything still had two commands that reached the network. Enforcing at this function covers both, plus any caller added later, and because redirects recurse into performHttpFetch each hop is checked too: an allowed host cannot redirect the request onward to a denied one.
const policyDenial = urlPolicyDenialReason(targetUrl, loadConfig().webfetch);
if (policyDenial !== null) {
rejectPromise(new Error(`${policyDenial}: ${truncateUrl(targetUrl)}`));
return;
}
// Node's http/https `lookup` option (ssrfPinnedLookup, passed to mod.request below) is only invoked when the hostname actually needs DNS resolution. For a literal IPv4 address Node connects directly and never calls the custom lookup function at all, silently bypassing SSRF protection for URLs like http://127.0.0.1/... or http://169.254.169.254/... (cloud metadata) -- on both the initial request and every redirect hop, since this function recurses into itself for redirects. Check literal IPs explicitly up front; there is no DNS to pin for them in the first place.
const literalIp = parsed.hostname.startsWith('[') && parsed.hostname.endsWith(']')
? parsed.hostname.slice(1, -1)
: parsed.hostname;
const isLiteralPrivate = isIPv4(literalIp) ? isPrivateIPv4(literalIp)
: isIPv6(literalIp) ? isPrivateIPv6(literalIp)
: false;
if (isLiteralPrivate) {
rejectPromise(new Error(`URL blocked by SSRF safety check: ${truncateUrl(targetUrl)}`));
return;
}
const remainingMs = opts.deadlineAt - Date.now();
if (remainingMs <= 0) {
rejectPromise(new Error(`Request timed out after ${opts.timeoutSec}s fetching ${truncateUrl(targetUrl)}`));
return;
}
const mod = parsed.protocol === 'https:' ? https : http;
let settled = false;
const req = mod.request({
hostname: parsed.hostname,
port: parsed.port === '' ? undefined : Number(parsed.port),
path: parsed.pathname + parsed.search,
method: 'GET',
lookup: ssrfPinnedLookup,
headers: { Host: parsed.host, ...opts.requestHeaders },
});
const deadlineTimer = setTimeout(() => {
if (settled) return;
settled = true;
req.destroy();
rejectPromise(new Error(`Request timed out after ${opts.timeoutSec}s fetching ${truncateUrl(targetUrl)}`));
}, remainingMs);
req.on('response', (res) => {
const status = res.statusCode ?? 0;
if (status >= 300 && status < 400 && res.headers.location) {
const location = res.headers.location;
res.resume();
if (settled) return;
settled = true;
clearTimeout(deadlineTimer);
if (opts.redirectsLeft <= 0) {
rejectPromise(new Error(`Too many redirects fetching ${truncateUrl(targetUrl)}`));
return;
}
let nextUrl: string;
let nextParsed: URL;
try {
nextParsed = new URL(location, targetUrl);
nextUrl = nextParsed.toString();
} catch {
rejectPromise(new Error(`Invalid redirect location fetching ${truncateUrl(targetUrl)}`));
return;
}
// Only forward requestHeaders to the redirect target when the origin is unchanged -- otherwise a caller-supplied Authorization/API-key header would silently leak to whatever cross-origin host the server redirects to. Protocol is part of that test: `host` is only host:port, so an https -> http redirect to the same host compared equal and kept the headers, putting the credential on the wire in cleartext at a destination the responding server chooses via Location. Comparing origin covers scheme, host and port together, which is what "same origin" means everywhere else.
const sameOrigin = nextParsed.origin === parsed.origin;
const nextOpts: HttpFetchOpts = {
...opts,
redirectsLeft: opts.redirectsLeft - 1,
requestHeaders: sameOrigin ? opts.requestHeaders : {},
};
performHttpFetch(nextUrl, nextOpts).then(resolvePromise, rejectPromise);
return;
}
const headers: Record<string, string> = {};
for (const [k, v] of Object.entries(res.headers)) {
if (typeof v === 'string') headers[k.toLowerCase()] = v;
else if (Array.isArray(v)) headers[k.toLowerCase()] = v.join(', ');
}
const chunks: Buffer[] = [];
let total = 0;
res.on('data', (chunk: Buffer) => {
if (settled) return;
total += chunk.length;
if (total > opts.maxSizeBytes) {
settled = true;
clearTimeout(deadlineTimer);
req.destroy();
rejectPromise(new Error(`File too large: ${total} bytes > ${opts.maxSizeBytes}`));
return;
}
chunks.push(chunk);
});
res.on('end', () => {
if (settled) return;
settled = true;
clearTimeout(deadlineTimer);
resolvePromise({ status, statusText: res.statusMessage ?? '', headers, body: Buffer.concat(chunks) });
});
res.on('error', (err) => {
if (settled) return;
settled = true;
clearTimeout(deadlineTimer);
rejectPromise(err);
});
});
req.on('error', (err) => {
if (settled) return;
settled = true;
clearTimeout(deadlineTimer);
rejectPromise(err);
});
req.end();
});
}