Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions packages/decepticon/decepticon/sandbox_web/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import subprocess
import tempfile
import time
from typing import Optional
from typing import Callable, Optional

from .fetch_chain import Attempt
from .validators import Verdict, validate
Expand Down Expand Up @@ -134,6 +134,7 @@ def run_playwright_fallback(
timeout: int = 90,
profile_dir: Optional[str] = None,
force_executor: Optional[str] = None,
scope_check: Optional[Callable[[str], bool]] = None,
) -> tuple[Attempt, str]:
"""Invoke the appropriate Playwright executor.

Expand Down Expand Up @@ -195,6 +196,7 @@ def run_playwright_fallback(
# template runs without xvfb; set INSANE_HEADLESS=0 (with xvfb present)
# to run headful, which evades headless-detecting WAFs (Akamai/DataDome).
"headless": os.environ.get("INSANE_HEADLESS", "1") not in ("0", "false", "no"),
"allowPrivate": False,

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Honor the private-network opt-in in Playwright

In environments that intentionally set INSANE_ALLOW_PRIVATE=1, the curl transport and the Python final-URL check use safety.allow_private_default(), but the browser template is always invoked with allowPrivate: false. That makes the new route guard abort loopback/RFC1918 requests before the Python-side opt-in can permit them, regressing local/private engagements that require Playwright fallback.

Useful? React with 👍 / 👎.

}
if choice == "playwright_mobile_chrome":
args["device"] = "iPhone 13 Pro"
Expand All @@ -213,13 +215,34 @@ def run_playwright_fallback(
# Fall back to treating raw stdout as HTML for forward/backward compat.
html, final_url, status, cookies, user_agent, automation = _parse_envelope(stdout, url)

from . import safety

final_url = final_url or url
ok, reason = safety.classify_url(final_url, allow_private=safety.allow_private_default())
if not ok:
att.status = status
att.body_size = len(html)
att.verdict = Verdict.BLOCKED.value
att.reasons = [f"ssrf_blocked:{reason}"]
att.error = f"ssrf_blocked:{reason}"
att.url = final_url
return att, ""
if scope_check is not None and not scope_check(final_url):

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Gate Playwright redirects before navigation

This RoE check runs only after _run_node_template() has already allowed Chromium to navigate and return the final page. With scope_check set, an in-scope URL that redirects to an out-of-scope public host will still be fetched by the browser before being marked blocked here, unlike the curl path's pre-hop gate; the scope decision needs to be enforced inside the Playwright route before route.continue().

Useful? React with 👍 / 👎.

att.status = status
att.body_size = len(html)
att.verdict = Verdict.BLOCKED.value
att.reasons = ["roe_out_of_scope"]
att.error = "ROE_REFUSED: final URL not in engagement scope"
att.url = final_url
return att, ""

resp = _FakeResp(html, status=status, final_url=final_url)
vr = validate(resp, success_selectors=success_selectors)
att.status = status
att.body_size = len(html)
att.verdict = vr.verdict.value
att.reasons = list(vr.reasons) + ([f"automation:{automation}"] if automation else [])
att.url = final_url or url
att.url = final_url

# Cookie bridge: a browser that cleared a JS challenge yields exactly the
# cookies + UA a plain HTTP client needs. Seed the curl_cffi pool so
Expand Down
7 changes: 7 additions & 0 deletions packages/decepticon/decepticon/sandbox_web/fetch_chain.py
Original file line number Diff line number Diff line change
Expand Up @@ -676,6 +676,12 @@ def _jitter():

# If a terminal-nonsuccess (404/auth/429) stopped us, browser won't help.
skip_browser = stop_reason in _TERMINAL_NONSUCCESS_VALUES
# Curl transport already refused a private/internal redirect hop. Do not
# re-try the same attacker-controlled URL with the browser tier, which would
# otherwise follow that redirect outside the curl SSRF guard.
if any((a.error or "").startswith("ssrf_redirect_blocked:") for a in trace):
skip_browser = True
stop_reason = stop_reason or "ssrf_redirect_blocked"

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Mark SSRF redirect blocks as terminal

When any curl attempt records ssrf_redirect_blocked: after the grid later sets stop_reason to "exhausted" or "budget", this assignment keeps that existing value, so _give_up() still reports normal exhaustion and advertises playwright_mcp as an untried escalation. In the exact scenario this guard is trying to suppress, the caller can be told to retry the attacker-controlled redirect in an agent browser; treat the SSRF redirect block as the terminal stop reason instead of only filling it when empty.

Useful? React with 👍 / 👎.


# -------- Phase 3: Playwright fallback ----------------------------------
if enable_playwright and not skip_browser:
Expand All @@ -697,6 +703,7 @@ def _jitter():
device_class=device_class,
force_executor=fb_name,
timeout=timeout if timeout and timeout > 30 else 90,
scope_check=scope_check,
)
trace.append(pw_attempt)
browser_used += 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"private": true,
"description": "Local deps for Playwright real-Chrome templates. npm install && npx playwright install chrome",
"dependencies": {
"ipaddr.js": "^2.2.0",
"playwright": "^1.58.2",
"playwright-extra": "^4.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
* NO-SITE-NAME RULE: same as playwright_real_chrome.js — no hostname branches.
*/

const dns = require('dns').promises;
const ipaddr = require('ipaddr.js');

function writeStdoutAsync(payload) {
return new Promise((resolve, reject) => {
process.stdout.write(payload, (err) => (err ? reject(err) : resolve()));
Expand Down Expand Up @@ -41,6 +44,61 @@ async function readStdinJson() {
});
}

function isBlockedIp(host) {
try {
let ip = ipaddr.process(host); // unwrap IPv4-mapped IPv6 addresses
if (ip.kind() === 'ipv6' && ip.match(ipaddr.parse('64:ff9b::'), 96)) {
const bytes = ip.toByteArray();
ip = ipaddr.fromByteArray(bytes.slice(12)); // NAT64 WKP embeds IPv4 in low 32 bits
}
const range = ip.range();
return !['unicast'].includes(range);
} catch (_e) {
return false;
}
}

async function classifyUrl(candidate, allowPrivate) {
let parsed;
try {
parsed = new URL(candidate);
} catch (e) {
return { ok: false, reason: `parse_error:${e.message || e}` };
}
if (!['http:', 'https:'].includes(parsed.protocol)) {
return { ok: false, reason: `scheme:${parsed.protocol.replace(':', '') || 'none'}` };
}
if (allowPrivate) {
return { ok: true, reason: 'allow_private' };
}
const host = parsed.hostname;
if (isBlockedIp(host)) {
return { ok: false, reason: `ip_blocked:${host}` };
}
try {
const records = await dns.lookup(host, { all: true });
for (const record of records) {
if (isBlockedIp(record.address)) {
return { ok: false, reason: `resolves_internal:${host}->${record.address}` };
}
}
} catch (_e) {
return { ok: false, reason: 'resolve_failed_blocked' };
}
return { ok: true, reason: 'public' };
}

async function installSafetyGuard(page, allowPrivate) {
await page.route('**/*', async (route) => {
const check = await classifyUrl(route.request().url(), allowPrivate);
if (!check.ok) {
await route.abort('blockedbyclient');
return;
}
await route.continue();
});
}

async function main() {
const args = await readStdinJson();
const url = args.url;
Expand All @@ -51,6 +109,7 @@ async function main() {
const waitSelector = args.waitSelector || null;
const timeoutMs = args.timeout || 60000;
const headless = args.headless ?? false;
const allowPrivate = args.allowPrivate === true;

let chromium, devices;
let automation = 'playwright';
Expand Down Expand Up @@ -90,6 +149,7 @@ async function main() {
...dev,
});
const page = await ctx.newPage();
await installSafetyGuard(page, allowPrivate);
const deadline = Date.now() + timeoutMs;
const rem = (cap) => Math.max(1000, Math.min(cap || timeoutMs, deadline - Date.now()));
const mainResp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: rem(90000) });
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/

const fs = require('fs');
const dns = require('dns').promises;
const ipaddr = require('ipaddr.js');

// Drain stdout fully before exiting. `process.exit()` can truncate a large
// HTML payload because it does not wait for pending stdout I/O (Node docs).
Expand Down Expand Up @@ -51,6 +53,61 @@ async function readStdinJson() {
});
}

function isBlockedIp(host) {
try {
let ip = ipaddr.process(host); // unwrap IPv4-mapped IPv6 addresses
if (ip.kind() === 'ipv6' && ip.match(ipaddr.parse('64:ff9b::'), 96)) {
const bytes = ip.toByteArray();
ip = ipaddr.fromByteArray(bytes.slice(12)); // NAT64 WKP embeds IPv4 in low 32 bits
}
const range = ip.range();
return !['unicast'].includes(range);
} catch (_e) {
return false;
}
}

async function classifyUrl(candidate, allowPrivate) {
let parsed;
try {
parsed = new URL(candidate);
} catch (e) {
return { ok: false, reason: `parse_error:${e.message || e}` };
}
if (!['http:', 'https:'].includes(parsed.protocol)) {
return { ok: false, reason: `scheme:${parsed.protocol.replace(':', '') || 'none'}` };
}
if (allowPrivate) {
return { ok: true, reason: 'allow_private' };
}
const host = parsed.hostname;
if (isBlockedIp(host)) {
return { ok: false, reason: `ip_blocked:${host}` };
}
try {
const records = await dns.lookup(host, { all: true });
for (const record of records) {
if (isBlockedIp(record.address)) {
return { ok: false, reason: `resolves_internal:${host}->${record.address}` };
}
}
} catch (_e) {
return { ok: false, reason: 'resolve_failed_blocked' };
}
return { ok: true, reason: 'public' };
}

async function installSafetyGuard(page, allowPrivate) {
await page.route('**/*', async (route) => {
const check = await classifyUrl(route.request().url(), allowPrivate);
if (!check.ok) {
await route.abort('blockedbyclient');
return;
}
await route.continue();
});
}

async function main() {
const args = await readStdinJson();
const url = args.url;
Expand All @@ -61,6 +118,7 @@ async function main() {
const timeoutMs = args.timeout || 60000;
const headless = args.headless ?? false; // Akamai/etc detect headless
const viewport = args.viewport || { width: 1366, height: 900 };
const allowPrivate = args.allowPrivate === true;

let chromium;
let automation = 'playwright';
Expand Down Expand Up @@ -110,6 +168,7 @@ async function main() {
}
ctx = await chromium.launchPersistentContext(profileDir, ctxOpts);
const page = await ctx.newPage();
await installSafetyGuard(page, allowPrivate);
// Single shared deadline across warmup + main + reload navigations so the
// first nav can't eat the whole budget and starve the rest.
const deadline = Date.now() + timeoutMs;
Expand Down
52 changes: 50 additions & 2 deletions packages/decepticon/tests/unit/sandbox_web/test_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from decepticon.sandbox_web.validators import SMALL_BODY_THRESHOLD, Verdict


def _envelope(html: str, final_url: str = "https://example.com/final") -> str:
def _envelope(html: str, final_url: str = "https://8.8.8.8/final") -> str:
return json.dumps(
{
"html": html,
Expand Down Expand Up @@ -69,10 +69,58 @@
)
assert att.executor == "playwright_real_chrome" # remapped, not MCP
assert att.verdict == Verdict.STRONG_OK.value
assert att.url == "https://example.com/final"
assert att.url == "https://8.8.8.8/final"
assert out == html


def test_final_private_url_is_blocked(monkeypatch: pytest.MonkeyPatch) -> None:
html = "x" * (SMALL_BODY_THRESHOLD + 50) + "<article id='c'>hi</article>"
monkeypatch.setattr(executor, "_chrome_channel_available", lambda: True)
monkeypatch.setattr(
executor,
"_run_node_template",
lambda template, args, timeout=90: (
0,
_envelope(html, final_url="http://127.0.0.1:9876/secret"),
"",
),
)
att, out = run_playwright_fallback(
"https://example.com/x",
profile_id="unknown_challenge",
success_selectors=["article#c"],
force_executor="playwright_real_chrome",
)
assert att.verdict == Verdict.BLOCKED.value
assert att.error == "ssrf_blocked:ip_blocked:127.0.0.1"
assert att.url == "http://127.0.0.1:9876/secret"
assert out == ""


def test_final_out_of_scope_url_is_blocked(monkeypatch: pytest.MonkeyPatch) -> None:
html = "x" * (SMALL_BODY_THRESHOLD + 50) + "<article id='c'>hi</article>"
monkeypatch.setattr(executor, "_chrome_channel_available", lambda: True)
monkeypatch.setattr(
executor,
"_run_node_template",
lambda template, args, timeout=90: (
0,
_envelope(html, final_url="https://8.8.4.4/final"),
"",
),
)
att, out = run_playwright_fallback(
"https://example.com/x",
profile_id="unknown_challenge",
success_selectors=["article#c"],
force_executor="playwright_real_chrome",
scope_check=lambda candidate: "example.com" in candidate,

Check failure

Code scanning / CodeQL

Incomplete URL substring sanitization High test

The string
example.com
may be at an arbitrary position in the sanitized URL.
)
assert att.verdict == Verdict.BLOCKED.value
assert att.error == "ROE_REFUSED: final URL not in engagement scope"
assert out == ""


def test_rendered_challenge_is_challenge(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(executor, "_chrome_channel_available", lambda: True)
monkeypatch.setattr(
Expand Down
Loading