diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..7ede2e5 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,22 @@ +{ + "name": "adaptive-fetch", + "version": "0.0.0", + "description": "Resilient, site-agnostic public-page reader for Claude Code — auto-bypasses blocked sites (403/WAF/CAPTCHA) via a Rust engine with browser TLS-fingerprint impersonation, a diversity-ordered fetch grid, and a no-give-up failure gate. No API keys.", + "author": { + "name": "Minsu Lee", + "url": "https://github.com/amondnet" + }, + "homepage": "https://github.com/pleaseai/adaptive-fetch", + "repository": "https://github.com/pleaseai/adaptive-fetch", + "license": "MIT", + "keywords": [ + "web-access", + "webfetch", + "bypass", + "waf", + "tls-impersonation", + "rust", + "playwright", + "scraping" + ] +} diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..45e3d02 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Report a problem to help us improve +title: "" +labels: ["bug"] +--- + +## Describe the bug + +A clear and concise description of what the bug is. + +## To reproduce + +Steps to reproduce the behavior: + +1. ... +2. ... +3. See error + +## Expected behavior + +What you expected to happen. + +## Environment + +- Package version: +- Runtime/OS: + +## Additional context + +Logs, screenshots, or anything else that helps. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..af9041d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Question or discussion + url: https://github.com/pleaseai/adaptive-fetch/discussions + about: Ask questions and discuss ideas here instead of opening an issue. + - name: Security vulnerability + url: https://github.com/pleaseai/adaptive-fetch/security/advisories/new + about: Report security issues privately — do not open a public issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..f1cf4e8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: Suggest an idea or improvement +title: "" +labels: ["enhancement"] +--- + +## Problem + +What problem does this feature solve? What is the use case? + +## Proposed solution + +A clear and concise description of what you want to happen. + +## Alternatives considered + +Other approaches you've thought about, and why this one is preferable. + +## Additional context + +Anything else that helps explain the request. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..733b8b9 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,17 @@ + + +## Summary + + + +## Related issue + + + +## Checklist + +- [ ] PR title follows Conventional Commits +- [ ] Tests added or updated, and the suite passes (`cargo test`) +- [ ] Lint/format pass (`mise run check`) +- [ ] Documentation updated if behavior changed +- [ ] No breaking change, or a `BREAKING CHANGE:` note is included diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2a56134 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + name: fmt + clippy + build + test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # Installs the toolchain pinned in mise.toml ([tools] rust + bun). + - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + - run: cargo fmt --all -- --check + - run: cargo clippy --all-targets -- -D warnings + - run: cargo build --locked + - run: cargo test --locked diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b7859cb --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# Rust build output +/target +**/*.rs.bk + +# Cargo.lock IS committed (this is a binary/application crate) — do not ignore it. + +# Downloaded prebuilt engine binary (M6 setup.sh fetches the platform asset here) +/skills/adaptive-fetch/engine/bin/ + +# Node deps for the Playwright fallback templates (M4) +node_modules/ + +# Environment / secrets +.env +.env.* +!.env.example + +# OS / editor +.DS_Store +*.swp + +# Claude Code local settings (internal marketplace config, telemetry labels) +.claude/settings.local.json diff --git a/.worktreeinclude b/.worktreeinclude new file mode 100644 index 0000000..4c44c43 --- /dev/null +++ b/.worktreeinclude @@ -0,0 +1,10 @@ +# Files copied into a new worktree (gitignored files matching these patterns). +# .gitignore syntax. Details: Skill("standards:dev-tooling") -> references/worktree-setup.md + +# Root environment / secrets +.env +.env.local +.env.*.local + +# Claude Code local settings (the symlink is gitignored via .git/info/exclude) +.claude/settings.local.json diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..c543ba4 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,132 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of + any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +**conduct@pleaseai.dev**. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][mozilla coc]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][faq]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[mozilla coc]: https://github.com/mozilla/diversity +[faq]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..cbbc5a9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,41 @@ +# Contributing + +Thanks for your interest in contributing! This guide covers how to get from a clone to a merged pull request. + +By participating, you agree to abide by our [Code of Conduct](./CODE_OF_CONDUCT.md). All documentation, code, comments, and commit messages in this repository are written in **English**. + +## Getting started + +```bash +git clone https://github.com/pleaseai/adaptive-fetch.git +cd adaptive-fetch +mise install # install pinned tool versions (rust + bun, see mise.toml) +cargo build # build the engine crate +``` + +## Development workflow + +1. Create a branch from the default branch (e.g. `feat/short-description` or `fix/issue-123`). +2. Make focused changes — keep each pull request to one logical change. +3. Run the checks below and make sure they pass. +4. Open a pull request and fill out the template. + +```bash +mise run check # fmt-check + clippy + test (mirrors CI) +cargo build # ensure it builds +``` + +## Commit messages + +We follow [Conventional Commits](https://www.conventionalcommits.org/): `type(scope): subject`, where `type` is one of `feat`, `fix`, `docs`, `refactor`, `test`, `chore`, etc. Breaking changes include a `BREAKING CHANGE:` footer. Versioning and the changelog are generated automatically from these messages, so accurate types matter. + +## Pull requests + +- Reference the issue your PR addresses (e.g. `Closes #123`). +- Use a Conventional-Commit-style PR title — it becomes the squash-merge commit. +- Make sure CI is green before requesting review. + +## Reporting bugs and requesting features + +Open an issue using the bug report or feature request template. For security +vulnerabilities, **do not** open a public issue — follow [SECURITY.md](./SECURITY.md). diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..3d909a9 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,249 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adaptive-fetch" +version = "0.0.0" +dependencies = [ + "clap", + "serde", + "serde_json", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..052a0c2 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +# Workspace root. The engine crate lives in engine-src/ so Rust sources stay +# separate from the Claude Code plugin packaging (skills/, references/, setup/). +[workspace] +resolver = "2" +members = ["engine-src"] + +[workspace.package] +edition = "2021" +license = "MIT" +repository = "https://github.com/pleaseai/adaptive-fetch" +authors = ["Minsu Lee "] diff --git a/LICENSE b/LICENSE index f3af155..8b22ac3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 pleaseai +Copyright (c) 2026 Passion Factory Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md new file mode 100644 index 0000000..e3a6c94 --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# adaptive-fetch + +[![CI](https://github.com/pleaseai/adaptive-fetch/actions/workflows/ci.yml/badge.svg)](https://github.com/pleaseai/adaptive-fetch/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](./LICENSE) + +A resilient, **site-agnostic** public-page reader for Claude Code. When a fetch +is blocked (402 / 403 / WAF / CAPTCHA), `adaptive-fetch` automatically tries every +bypass strategy until one works — no API keys, no proxy setup. + +Inspired by [`fivetaku/insane-search`](https://github.com/fivetaku/insane-search), +with the engine written in **Rust** so the core capability — browser TLS +fingerprint impersonation — runs natively, in-process, as a single static binary. + +> 🚧 **Status: M0 scaffold.** The CLI, plugin packaging, and toolchain are in +> place; the engine returns an honest "not implemented" result for now. The +> network stages land across milestones M1–M6. See the design RFC. + +## Design + +[`docs/rfcs/0001-adaptive-fetch.md`](docs/rfcs/0001-adaptive-fetch.md) — architecture, +invariants, and the milestone plan. + +## Repository layout + +``` +engine-src/ Rust crate (the engine binary) +skills/adaptive-fetch/ Claude Code skill (SKILL.md, references, profiles, templates) +docs/rfcs/ design RFCs +.claude-plugin/ plugin manifest +``` + +## Development + +```bash +mise install # rust + bun toolchain (see mise.toml) +mise run check # fmt-check + clippy + test (mirrors CI) +cargo run -p adaptive-fetch -- "https://example.com" --json +``` + +## License + +MIT diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..24b2c40 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,39 @@ +# Security Policy + +## Supported Versions + +Security fixes are provided for the latest released major version. Older +versions may receive fixes at the maintainers' discretion. + +| Version | Supported | +| ------- | ------------------ | +| latest | :white_check_mark: | +| older | :x: | + +## Reporting a Vulnerability + +**Please do not report security vulnerabilities through public GitHub issues, +discussions, or pull requests.** + +Instead, report them privately through GitHub's +[private vulnerability reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability): +go to the repository's **Security** tab and click **"Report a vulnerability"**. + +If you cannot use that channel, email **security@pleaseai.dev** instead. + +Please include: + +- A description of the vulnerability and its impact +- Steps to reproduce, or a proof of concept +- Affected versions, and any known mitigations + +## What to Expect + +- We aim to acknowledge your report within **72 hours**. +- We will keep you informed of progress toward a fix and may ask for additional + detail. +- Once a fix is released, we will credit you in the advisory unless you prefer + to remain anonymous. + +We ask that you give us a reasonable opportunity to address the issue before any +public disclosure. diff --git a/docs/rfcs/0001-adaptive-fetch.md b/docs/rfcs/0001-adaptive-fetch.md new file mode 100644 index 0000000..de8453b --- /dev/null +++ b/docs/rfcs/0001-adaptive-fetch.md @@ -0,0 +1,372 @@ +# RFC 0001 — adaptive-fetch engine + +| | | +|---|---| +| **Status** | Accepted (engine language) — implementation pending | +| **Author** | Minsu Lee (@amondnet) | +| **Supersedes** | — | + +> **Decision log:** engine language = **Rust**, confirmed. The Bun path (§9) is +> retained only as a documented alternative, not a live option. + +> A resilient public-page reader for Claude Code. When a fetch is blocked +> (402 / 403 / WAF / CAPTCHA), `adaptive-fetch` automatically tries every +> site-agnostic bypass strategy until one works — no API keys, no proxy setup. +> +> Modeled on [`fivetaku/insane-search`](https://github.com/fivetaku/insane-search), +> but with the engine rewritten in **Rust** so the core capability (browser TLS +> fingerprint impersonation) runs natively, in-process, as a single static binary. + +This RFC proposes the architecture and an implementation plan to review and +iterate on **before** any engine code is written. Follow-up RFCs (`0002-…`) can +refine individual milestones as the design evolves. + +--- + +## 1. Goal & scope + +`adaptive-fetch` is a **Claude Code plugin** (a `SKILL.md` + reference docs + +a compiled engine binary). When `WebFetch` returns a block, the skill kicks in +and routes the URL through an adaptive scheduler that escalates through cheap → +expensive bypass strategies and stops as soon as a *validated* success is found. + +It is **site-agnostic by construction**: the engine never hard-codes a domain, +selector, or brand name. Site knowledge enters only at runtime (caller-supplied +selectors / hints) or through generic WAF-product profiles. A small, sanctioned +exception exists for official public APIs (Phase 0 — see §4.3). + +### Non-goals +- Defeating authentication / paywalls (terminal failures, reported honestly). +- Bypassing rate limits by hammering (429 is transient → back off, not brute-force). +- Any per-site scraping logic baked into the engine (CI lint forbids it). + +--- + +## 2. Why Rust for the engine (the decisive trade-off) + +The entire value of insane-search is **TLS/JA3/JA4 fingerprint impersonation** — +making an HTTP client's TLS handshake look like a real Safari/Chrome so a WAF +that fingerprints the client doesn't reject it. insane-search gets this from +`curl_cffi` (a Python binding over `curl-impersonate`). + +A Bun/Node engine **cannot do this natively** — Node's TLS stack is itself a +recognizable fingerprint. So whichever language we pick, the impersonation layer +needs a native component. The comparison is therefore about *where* that native +code lives: + +| Concern | **Rust (recommended)** | Bun / TypeScript | +|---|---|---| +| TLS impersonation | `rquest` / `wreq` (BoringSSL) — native, in-process, Chrome/Safari/Firefox/Edge JA3+JA4+HTTP2 emulation. The Rust analogue of curl_cffi. | No native option. Must `bun:ffi`-load `tls-client` (Go `.so`/`.dylib`/`.dll`) or shell out to a `curl-impersonate` binary. Native dep either way. | +| Distribution | One self-contained static binary per platform (`ripgrep` model). | TS source + `bun install`, **plus** a per-platform native TLS lib to FFI-load or a sidecar process. | +| HTML/CSS selectors | `scraper` (servo's html5ever + selectors). | `cheerio` / `linkedom` — easy. | +| Startup latency | Cold-start ~ms; matters for a per-request CLI. | Bun cold-start fine, but FFI lib load adds setup. | +| pleaseai standard | Diverges (standard is bun/TS), but justified by capability. | Matches the standard toolchain. | + +**Decision: Rust (confirmed).** The one hard requirement (native TLS impersonation) +is in-process and first-class in Rust via `rquest`/`wreq`, and the output is a +single static binary that the skill invokes exactly like insane-search invokes +`python3 -m engine`. Bun would still need a per-platform native blob for the only +thing that actually matters, giving up its main advantage (pure-TS distribution) +while keeping a worse impersonation story. + +> The Bun path remains viable if avoiding per-platform binary distribution is the +> top priority — see §9 for the fallback design. This document assumes Rust. + +> Source-driven note: `rquest` was renamed to `wreq`; confirm the current crate +> name, version, and the exact `Emulation`/`Impersonate` profile API at +> implementation time rather than trusting this doc's snapshot. + +--- + +## 3. Deliverable shape (plugin layout) + +Mirrors insane-search's plugin structure; engine is a Rust binary instead of a +Python package. + +``` +adaptive-fetch/ +├─ .claude-plugin/ +│ └─ plugin.json # plugin manifest +├─ skills/ +│ └─ adaptive-fetch/ +│ ├─ SKILL.md # harness rules R1–R7 + Phase 0 index + usage +│ ├─ references/ # on-demand deep docs (tls, playwright, apis…) +│ │ ├─ tls-impersonate.md +│ │ ├─ playwright.md +│ │ ├─ public-api.md +│ │ ├─ json-api.md +│ │ ├─ media.md +│ │ └─ … (rss, metadata, cache-archive, naver, twitter, fallback) +│ └─ engine/ +│ ├─ bin/ # downloaded prebuilt binary lands here +│ ├─ waf_profiles.yaml # WAF-product priors (shipped, editable) +│ └─ templates/ # Playwright node templates (real-chrome, mobile) +├─ engine-src/ # Rust crate (the source of the binary) +│ ├─ Cargo.toml +│ └─ src/ +│ ├─ main.rs # CLI entrypoint + JSON output +│ ├─ lib.rs # `fetch(url, opts) -> FetchResult` +│ ├─ scheduler.rs # diversity planner + grid + failure gate (R6) +│ ├─ transport.rs # rquest session pool, warmup, cookie bridge +│ ├─ validators.rs # 4-layer validation, Verdict enum +│ ├─ waf_detector.rs # ranked WAF-product detection +│ ├─ url_transforms.rs # generic URL rewrite rules +│ ├─ phase0.rs # official public-API router (sanctioned) +│ ├─ executor.rs # Playwright fallback routing (node subprocess / MCP flag) +│ ├─ learning.rs # per-host winning-route store +│ └─ safety.rs # SSRF guard (private/loopback/metadata block-list) +├─ setup/ +│ └─ setup.sh # first-run: download correct binary, deps check +├─ orca.yaml # pleaseai worktree/dev-tooling (rust+mise variant) +├─ .worktreeinclude +└─ DESIGN.md +``` + +### Engine public contract (unchanged from insane-search, so SKILL.md rules port verbatim) + +```bash +adaptive-fetch "" [--selector ""] [--device auto|desktop|mobile] [--trace] [--json] +# exit 0 = ok (validated success), exit 1 = fail (with untried_routes in --json) +``` + +Library form (`fetch`) returns a `FetchResult` with the same fields insane-search +exposes: `ok`, `content`, `final_url`, `verdict`, `profile_used`, `trace[]`, +`planned_attempts`, `executed_attempts`, `grid_exhausted`, `stop_reason`, +`untried_routes[]`, `must_invoke_playwright_mcp`. + +--- + +## 4. Engine architecture (port of the insane-search invariants) + +The whole point of insane-search's engine is a set of **hard invariants** that +stop an agent from bailing out early. We carry these over exactly; only the +language changes. + +### 4.1 Single entrypoint + explicit phases + +`fetch()` is the only public API, but internally it is staged so `trace[]` can +attribute every attempt: + +``` +probe → validate → detect → plan → execute (grid) → fallback → report +``` + +### 4.2 Validation: HTTP 200 is the *start* of inspection, not success + +Port `validators.rs` as a layered AND check (insane-search's `validators.py`): + +1. **Status semantics** — 429 → `RateLimited` (transient), 401/407 → `AuthRequired` + (terminal), 404/410 → `NotFound` (terminal), 5xx → `Blocked`, 0 → `Unknown`. +2. **Hard challenge markers** — structural WAF containers (`Just a moment...`, + `sec-if-cpt-container`, Incapsula/Akamai strings) → `Challenge`, decisive. +3. **Size fingerprint** — body byte-size near a known-bad WAF stub size → `Challenge`. +4. **JSON awareness** — small non-empty parseable JSON is a *success* (`WeakOk`), + not a challenge (this is what unlocks the R7 API-first route). +5. **Caller positive proof** — `success_selectors` match → `StrongOk` + (`scraper` crate for CSS); requested-but-unmatched → `Challenge`. +6. **Heuristics (no proof)** — soft markers, tiny incomplete body, unresolved + Akamai `_abck=~-1~` sensor cookie → `SuspectOk` (non-terminal: keep searching). + +Verdict enum: `StrongOk`, `WeakOk`, `SuspectOk`(non-terminal), `Challenge`, +`Blocked`, `RateLimited`, `AuthRequired`, `NotFound`, `Unknown`. Terminal-nonsuccess += {`AuthRequired`, `NotFound`, `RateLimited`} (stop the TLS grid — more handshakes won't help). + +### 4.3 Phase 0 — official public-API router (the only site-aware module) + +`phase0.rs` is the **sole** engine file allowed to name platform hosts (exempt +from the bias linter). For recognised platforms it tries the official no-auth +endpoint *before* the generic grid: + +- **Reddit** → `.rss` (the JSON API is WAF-gated; RSS survives). +- **X/Twitter** → `cdn.syndication.twimg.com/tweet-result` + oEmbed (single + tweet), `syndication.twitter.com` timeline (profile, retry-once). +- **YouTube** → `yt-dlp --dump-json` subprocess. + +Extensible to HN/Bluesky/Mastodon/arXiv/etc. as documented in `references/`. + +### 4.4 WAF detection → ranked priors + +`waf_detector.rs` scores each profile in `waf_profiles.yaml` against the live +response (cookies / headers / server / body markers) and returns a **ranked +list** of `(profile_id, confidence)` — never a single verdict (a wrong single +guess cascades into a wrong plan). Profiles cover Akamai Bot Manager, Cloudflare +Turnstile, F5 BIG-IP, AWS WAF, DataDome, PerimeterX/HUMAN, plus an +`unknown_challenge` safety net. Each profile carries: detectors, capabilities +needed (`needs_real_tls_stack`, `needs_js_exec`, `needs_mobile_context`), TLS +candidate families, referer strategies, URL-transform order, and fallback chain. + +### 4.5 The diversity scheduler (the heart) + +`scheduler.rs` ports the v2 planner from `fetch_chain.py`: + +- Materialize the full grid = `url_transforms × tls_impersonate × referer` + across the top-3 detected profiles (round-robin interleaved so a confident #1 + can't starve #2/#3). +- **Order for diversity:** vary TLS *family* fastest, then transform, then + version depth, then referer — so a small attempt budget still touches every + family/transform instead of burning out on one family's old versions. +- `tls_impersonate_avoid` targets are **deprioritized, never deleted** (still + tried in exhaustive mode). +- `device_class` shapes the grid (`mobile` → mobile TLS + `m.` subdomain + transforms; `desktop` → desktop TLS only). +- Default `max_attempts = None` = **exhaustive** (honours R6). A numeric cap is + a *budget*; budget vs exhaustion vs early-terminal is reported via + `stop_reason` / `grid_exhausted` so a truncated run is never mistaken for a + true exhaustive failure. +- Jitter sleep only on a *continuing* (failed) attempt, never before returning success. + +### 4.6 Failure gate (R6) — give-up is never silent + +When `ok=false`, the engine reports what it could *not* itself do: +`untried_routes[]` and `must_invoke_playwright_mcp`. A non-terminal failure +always surfaces: (a) re-run exhaustive if grid not exhausted, (b) Playwright MCP +from the agent session (engine can only spawn local Node Chrome, so MCP is +structurally the agent's job), (c) `user_hint` retry. The CLI prints a +`⛔ NOT EXHAUSTED` block to stderr so the agent knows it must continue. 429 is +explicitly **not** terminal (back off + retry / different TLS / MCP). + +### 4.7 Transport: session pool + warmup + cookie bridge + +`transport.rs` ports `transport.py` onto `rquest`: + +- **Per-(host, impersonate) session pool** — reuse cookies (WAF sensors like + `_abck`, `cf_clearance` need to *mature* across requests) and the warm + connection across attempts and across pages of the same host. +- **Root warmup** — for deep URLs, hit the site root once so a sensor sets a + resolved cookie before the deep request (classic first-hit-rejection fix). +- **Browser→curl cookie bridge** — when a Playwright pass clears a JS challenge, + inject its cookies + UA into the pooled session so one expensive browser pass + converts into cheap impersonated-HTTP throughput (the FlareSolverr pattern). +- **SSRF guard** — `safety.rs` validates the initial URL *and every redirect hop* + against a private/loopback/link-local/cloud-metadata block-list; redirects are + followed manually so each hop is checked. + +### 4.8 Playwright fallback (capability-matched) + +`executor.rs` reads the profile's `capabilities_needed` and routes: + +| Capability | Executor | In the Rust engine | +|---|---|---| +| `needs_real_tls_stack` + `needs_js_exec` | local Node `playwright_real_chrome.js` | engine spawns `node` subprocess (templates shipped in `engine/templates/`) | +| `needs_js_exec` only | Playwright **MCP** | engine **cannot** drive MCP → sets `must_invoke_playwright_mcp=true`; the *agent* runs it (per R6) | +| `needs_mobile_context` | `playwright_mobile_chrome.js` | node subprocess, mobile device emulation | + +The Node templates are reused as-is from insane-search (no Rust browser needed); +the engine only orchestrates the subprocess and bridges cookies back. + +### 4.9 Self-learning (per-host winning route) + +`learning.rs` ports `learning.py`: a bounded, self-pruning JSON store mapping a +host (+ device class) to its last winning route `{transform, impersonate, +referer}`. On the next fetch the winning route is promoted to the probe identity +and the front of the grid; two consecutive *real* failures evict it. Any error in +the store is swallowed — learning can never break a fetch +(`ADAPTIVE_FETCH_LEARN=0` to disable). + +--- + +## 5. Crate / dependency choices (Rust) + +| Need | Crate | Note | +|---|---|---| +| TLS impersonation HTTP client | `rquest` / `wreq` | BoringSSL JA3/JA4/HTTP2 browser emulation — the core. Verify current name/API. | +| Async runtime | `tokio` | required by the client. | +| CSS selectors / HTML | `scraper` | for `success_selectors` validation. | +| WAF profiles (YAML) | `serde_yaml` (+ `serde`) | embed via `include_str!` with a ship-alongside override path. | +| JSON I/O (`--json`, learning store) | `serde_json` | result + trace serialization. | +| IP / CIDR classification (SSRF) | `ipnet`, `std::net` | block private/loopback/link-local/metadata. | +| CLI args | `clap` | `--selector/--device/--trace/--json`. | +| URL parsing | `url` | transforms + redirect resolution. | + +Map insane-search's impersonate target names (`safari`, `safari_ios`, +`chrome131`, `chrome_android`, `firefox`, `edge99`…) onto the chosen crate's +emulation profiles in `transport.rs`. Where a target has no 1:1 emulation, pick +the nearest and record the mapping in `references/tls-impersonate.md`. + +--- + +## 6. Distribution & install (the real cost of the Rust choice) + +The skill needs the right binary on the user's machine with zero friction. + +- **Build:** `cargo-dist` (or a `cross` matrix in CI) produces binaries for + `aarch64-apple-darwin`, `x86_64-apple-darwin`, `x86_64-unknown-linux-gnu`, + `aarch64-unknown-linux-gnu`, `x86_64-pc-windows-msvc`, published to GitHub + Releases with checksums. +- **Install:** `setup/setup.sh` (run once by SKILL.md Step 0, idempotent) + detects OS/arch, downloads the matching binary into `engine/bin/`, verifies the + checksum, `chmod +x`. Falls back to `cargo install` only if a Rust toolchain + is present and no prebuilt asset matches. +- **Alternative:** npm package with `optionalDependencies` per-platform (the + `esbuild`/`@biomejs` model) so `bunx`/`npx` resolves the right binary — + attractive because it matches the pleaseai bun toolchain for the *wrapper* + even though the engine is Rust. +- **Runtime deps the binary can't bundle:** `node` + `playwright` (only when the + Playwright fallback fires), `yt-dlp` (only for Phase 0 media). The skill checks + and installs these lazily, exactly like insane-search. + +CI must pin third-party GitHub Actions to full commit SHAs (pleaseai +`github-actions` rule). + +--- + +## 7. The "no site names" invariant + CI gate + +Port `bias_check.py` as a Rust test or a small CI script that fails the build if +any file under `engine-src/src/**` (except `phase0.rs`) or `waf_profiles.yaml` +contains a site domain, brand name, or `if url.contains("")` branch. +Allowed: descriptive prose in `SKILL.md`/`references/*.md`, the Phase 0 official +endpoints, runtime `success_selectors`/`user_hint`, and append-only observation logs. + +Boundary rule: *"Would this entry be valid on any other site running the same +WAF?"* → yes ⇒ `waf_profiles.yaml`; no ⇒ runtime hint. + +--- + +## 8. Implementation milestones (proposed, after this design is approved) + +| Milestone | Deliverable | Verifies | +|---|---|---| +| **M0 Scaffolding** | Cargo crate, `clap` CLI skeleton, `orca.yaml` (rust+mise), `.worktreeinclude`, CI (build + SHA-pinned actions), `plugin.json` | `cargo build`, plugin loads | +| **M1 Probe + validate** | `transport.rs` (single rquest GET + SSRF), `validators.rs` (all layers), `safety.rs`; CLI returns verdict + `--json` | unit tests on canned responses; example.com, a JSON API | +| **M2 Grid scheduler** | `waf_detector.rs`, `waf_profiles.yaml`, `url_transforms.rs`, `scheduler.rs` (diversity plan, exhaustive grid, failure gate, jitter), session pool + warmup | grid ordering tests; `untried_routes`/`grid_exhausted` correctness | +| **M3 Phase 0** | `phase0.rs` (reddit/x/youtube routers) | per-platform route tests, trace records | +| **M4 Playwright fallback** | `executor.rs` capability matching, Node templates, `must_invoke_playwright_mcp` flag, cookie bridge | mock subprocess; flag set correctly for MCP-only profiles | +| **M5 Learning** | `learning.rs` per-host store, eviction | promote/strike/evict tests | +| **M6 Skill + ship** | `SKILL.md` (R1–R7, Phase 0 index, intent table), `references/*.md`, `setup.sh`, release pipeline, bias-check CI, coverage battery | end-to-end against a real blocked site; bias gate green | + +Each milestone keeps files ≤500 LOC (engineering standard) — `scheduler.rs` is +the risk; split planner / executor-loop / result-builder if it grows. + +--- + +## 9. Bun fallback design (if per-platform binaries are rejected) + +If the team prefers to stay on the standard bun/TS toolchain and accept a native +TLS sidecar: + +- **TLS layer:** `bun:ffi` loading bogdanfinn's `tls-client` shared library + (Go, exposes JA3 impersonation), or a long-lived `curl-impersonate` / + `tls-client` HTTP sidecar the TS engine talks to. Per-platform native lib still + downloaded by `setup.sh`. +- **Everything else** maps cleanly to TS: `cheerio` (selectors), `yaml` + (profiles), native `fetch`/`undici` only for the non-impersonated Phase 0/JSON + routes, `node:net`/`ipaddr.js` (SSRF), Playwright node templates reused directly. +- **Cost:** the only hard requirement (impersonation) is the clunkiest part, and + we still ship a native blob — i.e. we pay Rust's distribution cost without + Rust's clean in-process impersonation. This is why §2 recommends Rust. + +--- + +## 10. Open decisions for review + +1. ~~**Engine language:** Rust vs Bun fallback (§9).~~ **RESOLVED → Rust.** +2. **Distribution:** GitHub Releases + `setup.sh` download, vs npm + `optionalDependencies` per-platform. +3. **Phase 0 breadth at v1:** ship reddit/x/youtube only, or also HN / Bluesky / + arXiv / Naver from the start. +4. **Playwright in v1:** include the local-Node fallback in the first release, or + ship curl-grid-only first and add browser fallback in a follow-up (relying on + the `must_invoke_playwright_mcp` agent route until then). +5. **Repo standards:** confirm `orca.yaml` should use the `mise` rust toolchain + variant (vs the default bun template). diff --git a/engine-src/Cargo.toml b/engine-src/Cargo.toml new file mode 100644 index 0000000..12d7899 --- /dev/null +++ b/engine-src/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "adaptive-fetch" +version = "0.0.0" +description = "Resilient, site-agnostic public-page reader: auto-bypasses blocked sites via browser TLS-fingerprint impersonation." +edition.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[[bin]] +name = "adaptive-fetch" +path = "src/main.rs" + +[lib] +name = "adaptive_fetch" +path = "src/lib.rs" + +[dependencies] +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +# Engine dependencies land as their milestones are implemented (RFC 0001 §5): +# M1: rquest/wreq (TLS impersonation), tokio, scraper, ipnet +# M2: serde_yaml (WAF profiles), url +# Kept out of M0 so the scaffold compiles fast and crate choices are pinned +# against real usage rather than guessed up front. diff --git a/engine-src/src/executor.rs b/engine-src/src/executor.rs new file mode 100644 index 0000000..b1d10e2 --- /dev/null +++ b/engine-src/src/executor.rs @@ -0,0 +1,11 @@ +//! Playwright fallback routing (M4, RFC 0001 §4.8). +//! +//! Reads a profile's `capabilities_needed` and routes: +//! +//! * needs_real_tls_stack + needs_js_exec → spawn local node `playwright_real_chrome.js` +//! * needs_js_exec only → set `must_invoke_playwright_mcp` (agent drives MCP) +//! * needs_mobile_context → `playwright_mobile_chrome.js` +//! +//! The engine never drives MCP itself — that is structurally the agent's job (R6). +// +// TODO(M4): capability matching, node subprocess orchestration, cookie bridge-back. diff --git a/engine-src/src/learning.rs b/engine-src/src/learning.rs new file mode 100644 index 0000000..91dbc2d --- /dev/null +++ b/engine-src/src/learning.rs @@ -0,0 +1,8 @@ +//! Self-learning: per-host winning-route store (M5, RFC 0001 §4.9). +//! +//! A bounded, self-pruning JSON store mapping (host, device_class) → last winning +//! route {transform, impersonate, referer}. Promoted to the front of the next +//! grid; evicted after two consecutive *real* failures. Any store error is +//! swallowed — learning can never break a fetch (ADAPTIVE_FETCH_LEARN=0 disables). +// +// TODO(M5): lookup(), record_success(), record_failure(), eviction, store I/O. diff --git a/engine-src/src/lib.rs b/engine-src/src/lib.rs new file mode 100644 index 0000000..a0a00ca --- /dev/null +++ b/engine-src/src/lib.rs @@ -0,0 +1,92 @@ +//! # adaptive-fetch engine +//! +//! A resilient, **site-agnostic** public-page reader. When a fetch is blocked +//! (402 / 403 / WAF / CAPTCHA) it escalates through cheap → expensive bypass +//! strategies and stops as soon as a *validated* success is found. +//! +//! Single public entrypoint: [`fetch`]. Internally staged so [`FetchResult`]'s +//! `trace` can attribute every attempt: +//! +//! ```text +//! probe → validate → detect → plan → execute (grid) → fallback → report +//! ``` +//! +//! Full design: `docs/rfcs/0001-adaptive-fetch.md`. +//! +//! **No-site-name invariant:** no module here (except `phase0`) may name a site +//! domain, selector, or brand. Site knowledge enters only via [`FetchOptions`]. + +mod options; +mod result; + +// Engine stages (RFC 0001 §4). Stubs now; filled in across milestones M1–M5. +mod executor; // M4: Playwright fallback routing (node subprocess / MCP flag) +mod learning; // M5: per-host winning-route store +mod phase0; // M3: official public-API router (the sole site-aware module) +mod safety; // M1: SSRF guard (private/loopback/link-local/metadata block-list) +mod scheduler; // M2: diversity planner + grid + failure gate (R6) +mod transport; // M1: rquest session pool, root warmup, browser→curl cookie bridge +mod url_transforms; // M2: generic URL rewrite rules +mod validators; // M1: 4-layer validation, Verdict classification +mod waf_detector; // M2: ranked WAF-product detection + +pub use options::{DeviceClass, FetchOptions, UserHint}; +pub use result::{Attempt, FetchResult, Verdict}; + +/// Fetch `url`, bypassing blocks site-agnostically. +/// +/// **M0 scaffold:** the engine stages are not implemented yet, so this returns +/// an honest "not yet implemented" [`FetchResult`] — `ok = false`, +/// `stop_reason = "unimplemented"` — with the remaining milestones surfaced in +/// `untried_routes`. This lets the CLI and plugin wiring be exercised before the +/// network stages land in M1+. +pub fn fetch(url: &str, _opts: &FetchOptions) -> FetchResult { + FetchResult { + ok: false, + content: String::new(), + final_url: url.to_string(), + verdict: Verdict::Unknown, + profile_used: None, + trace: Vec::new(), + summary: "engine not implemented yet (M0 scaffold)".to_string(), + planned_attempts: 0, + executed_attempts: 0, + grid_exhausted: false, + stop_reason: "unimplemented".to_string(), + untried_routes: vec![ + "M1: probe + 4-layer validation + SSRF transport".to_string(), + "M2: diversity grid scheduler + WAF detection".to_string(), + "M3: Phase 0 official-API router".to_string(), + "M4: Playwright fallback".to_string(), + ], + must_invoke_playwright_mcp: false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scaffold_returns_honest_unimplemented_result() { + let r = fetch("https://example.com", &FetchOptions::default()); + assert!(!r.ok); + assert_eq!(r.stop_reason, "unimplemented"); + assert_eq!(r.final_url, "https://example.com"); + assert!(!r.untried_routes.is_empty()); + } + + #[test] + fn verdict_terminality() { + assert!(Verdict::StrongOk.is_ok()); + assert!(!Verdict::SuspectOk.is_ok()); + assert!(Verdict::NotFound.is_terminal_nonsuccess()); + assert!(!Verdict::Challenge.is_terminal_nonsuccess()); + } + + #[test] + fn defaults_are_exhaustive() { + // None == exhaustive (R6): the default must never silently cap attempts. + assert_eq!(FetchOptions::default().max_attempts, None); + } +} diff --git a/engine-src/src/main.rs b/engine-src/src/main.rs new file mode 100644 index 0000000..318a4ec --- /dev/null +++ b/engine-src/src/main.rs @@ -0,0 +1,105 @@ +//! `adaptive-fetch` CLI — the single entrypoint the Claude Code skill invokes. +//! +//! Usage: +//! ```text +//! adaptive-fetch "" [--selector ""]... [--device auto|desktop|mobile] [--trace] [--json] +//! ``` +//! Exit code: `0` = validated success, `1` = failure (with `untried_routes`). + +use std::process::ExitCode; + +use adaptive_fetch::{fetch, DeviceClass, FetchOptions}; +use clap::Parser; + +#[derive(Parser)] +#[command( + name = "adaptive-fetch", + version, + about = "Resilient site-agnostic page reader — auto-bypasses blocked sites." +)] +struct Cli { + /// URL to fetch. + url: String, + + /// CSS selector proving success (repeatable). Strongest positive proof. + #[arg(long = "selector", value_name = "CSS")] + selectors: Vec, + + /// Device shaping for the impersonation grid. + #[arg(long, value_enum, default_value_t = DeviceClass::Auto)] + device: DeviceClass, + + /// Print the per-attempt trace to stderr. + #[arg(long)] + trace: bool, + + /// Emit the full result (metadata + trace) as JSON to stdout. + #[arg(long)] + json: bool, +} + +fn main() -> ExitCode { + let cli = Cli::parse(); + + let opts = FetchOptions { + success_selectors: cli.selectors, + device_class: cli.device, + ..FetchOptions::default() + }; + + let result = fetch(&cli.url, &opts); + + if cli.json { + // content is excluded from serialization (see FetchResult::content). + match serde_json::to_string_pretty(&result) { + Ok(s) => println!("{s}"), + Err(e) => { + eprintln!("adaptive-fetch: failed to serialize result: {e}"); + return ExitCode::FAILURE; + } + } + } else { + if !result.content.is_empty() { + print!("{}", result.content); + } + eprintln!( + "verdict={:?} stop={} profile={} attempts={}/{} content={}B", + result.verdict, + result.stop_reason, + result.profile_used.as_deref().unwrap_or("-"), + result.executed_attempts, + result.planned_attempts, + result.content_length(), + ); + if cli.trace { + for (i, a) in result.trace.iter().enumerate() { + eprintln!( + " [{i}] {phase} {exec} {imp} {tf} referer:{rf} -> {verdict:?} ({status})", + phase = a.phase, + exec = a.executor, + imp = a.impersonate.as_deref().unwrap_or("-"), + tf = a.url_transform, + rf = a.referer, + verdict = a.verdict, + status = a.status, + ); + } + } + if !result.ok && !result.untried_routes.is_empty() { + // R6: give-up is never silent — the agent must continue these routes. + eprintln!("\n\u{26d4} NOT EXHAUSTED — untried routes remain:"); + for route in &result.untried_routes { + eprintln!(" - {route}"); + } + if result.must_invoke_playwright_mcp { + eprintln!(" ! Playwright MCP must be driven from the agent session."); + } + } + } + + if result.ok { + ExitCode::SUCCESS + } else { + ExitCode::FAILURE + } +} diff --git a/engine-src/src/options.rs b/engine-src/src/options.rs new file mode 100644 index 0000000..19cc0f5 --- /dev/null +++ b/engine-src/src/options.rs @@ -0,0 +1,59 @@ +//! Caller-facing fetch options (RFC 0001 §3, §4.5). +//! +//! Site knowledge enters the engine only through these options +//! (`success_selectors`, `user_hint`) — never through hard-coded site logic. + +use clap::ValueEnum; + +/// Device shaping for the impersonation grid (RFC 0001 §4.5). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, ValueEnum)] +#[value(rename_all = "lower")] +pub enum DeviceClass { + /// Follow the detected profile's strategy. + #[default] + Auto, + /// Desktop TLS only; disable `m.` subdomain transforms. + Desktop, + /// Mobile TLS only; enable `m.` subdomain transforms. + Mobile, +} + +/// Optional runtime hints. Applied to the current call only — never persisted +/// by the engine (the self-learning store in `learning.rs` is separate). +#[derive(Debug, Clone, Default)] +pub struct UserHint { + /// TLS impersonation target to try first (e.g. `safari_ios`, `chrome`). + pub impersonate_first: Option, + /// Referer strategy override (`self_root` | `google_search` | `none`). + pub referer_strategy: Option, +} + +/// Inputs to [`crate::fetch`]. +#[derive(Debug, Clone)] +pub struct FetchOptions { + /// CSS selectors proving a real page rendered (strongest positive proof). + pub success_selectors: Vec, + pub device_class: DeviceClass, + pub user_hint: Option, + pub timeout_secs: u64, + /// `None` = exhaustive (honours R6); `Some(n)` = total attempt budget. + pub max_attempts: Option, + pub enable_playwright: bool, + pub enable_phase0: bool, + pub enable_learning: bool, +} + +impl Default for FetchOptions { + fn default() -> Self { + Self { + success_selectors: Vec::new(), + device_class: DeviceClass::Auto, + user_hint: None, + timeout_secs: 25, + max_attempts: None, // exhaustive by default (R6) + enable_playwright: true, + enable_phase0: true, + enable_learning: true, + } + } +} diff --git a/engine-src/src/phase0.rs b/engine-src/src/phase0.rs new file mode 100644 index 0000000..af1f85a --- /dev/null +++ b/engine-src/src/phase0.rs @@ -0,0 +1,8 @@ +//! Phase 0 — official public-API router (M3, RFC 0001 §4.3). +//! +//! The SANCTIONED exception to the no-site-name rule: the ONLY module allowed to +//! name platform hosts, and the only one exempt from the bias linter. Tries an +//! official no-auth endpoint BEFORE the generic grid (Reddit→.rss, +//! X→tweet-result/oEmbed/syndication, YouTube→yt-dlp). +// +// TODO(M3): detect(url) -> platform, per-platform routers, route() entrypoint. diff --git a/engine-src/src/result.rs b/engine-src/src/result.rs new file mode 100644 index 0000000..b604ec4 --- /dev/null +++ b/engine-src/src/result.rs @@ -0,0 +1,110 @@ +//! Public result + verdict types — the stable engine contract. +//! +//! The field set mirrors insane-search's `FetchResult` so the `SKILL.md` +//! harness rules (notably the R6 failure gate) port verbatim. See +//! `docs/rfcs/0001-adaptive-fetch.md` §3, §4.2, §4.6. + +use serde::{Deserialize, Serialize}; + +/// Classification of a fetched response. +/// +/// HTTP 200 is the *start* of inspection, not success (RFC 0001 §4.2): a +/// response is only a success once it clears the layered validation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Verdict { + /// Caller-supplied positive proof matched → terminal success. + StrongOk, + /// Clean response, no negative signal → terminal success. + WeakOk, + /// Ambiguous (unresolved sensor cookie / soft marker) → NON-terminal: + /// kept as best-effort while the grid keeps searching for real proof. + SuspectOk, + /// WAF challenge (negative proof). + Challenge, + /// Generic non-2xx block. + Blocked, + /// 429 — back off, do not hammer. Transient, NOT a terminal wall. + RateLimited, + /// 401/407 — terminal; retrying TLS cannot help. + AuthRequired, + /// 404/410 — terminal. + NotFound, + /// Exception / dependency missing / unscored. + Unknown, +} + +impl Verdict { + /// Terminal success only. `SuspectOk` is intentionally excluded. + pub fn is_ok(self) -> bool { + matches!(self, Verdict::StrongOk | Verdict::WeakOk) + } + + /// "Stop the grid — more TLS attempts cannot help." 429 is included here + /// (don't hammer) but is surfaced as a *transient* route by the failure gate. + pub fn is_terminal_nonsuccess(self) -> bool { + matches!( + self, + Verdict::AuthRequired | Verdict::NotFound | Verdict::RateLimited + ) + } +} + +/// One attempt in the trace: a (transform × impersonate × referer × executor) +/// combination and how the response was judged. Exposed so callers can diagnose +/// a failure without re-running. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Attempt { + /// `probe` | `grid` | `fallback` | `phase0`. + pub phase: String, + /// `rquest` | `playwright_real_chrome` | `phase0:` | ... + pub executor: String, + pub url: String, + /// `original` | `mobile_subdomain` | `am_prefix` | `drop_www` | `-`. + pub url_transform: String, + /// TLS impersonation target (None for non-curl executors). + pub impersonate: Option, + pub referer: String, + pub status: u16, + pub body_size: usize, + pub verdict: Verdict, + pub reasons: Vec, + pub elapsed_ms: u64, + pub error: Option, +} + +/// The single value [`crate::fetch`] returns. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FetchResult { + pub ok: bool, + /// Fetched body. TODO(M1): exclude from `--json` (emit `content_length` + /// instead) so large HTML never bloats the trace output. + #[serde(skip)] + pub content: String, + pub final_url: String, + pub verdict: Verdict, + pub profile_used: Option, + pub trace: Vec, + pub summary: String, + // --- scheduler diagnostics (RFC 0001 §4.5) --- + pub planned_attempts: u32, + pub executed_attempts: u32, + pub grid_exhausted: bool, + /// `success` | `exhausted` | `budget` | `` | `error`. + pub stop_reason: String, + // --- failure gate (R6, RFC 0001 §4.6) --- + /// When `ok == false`, the escalation routes the engine could not perform + /// itself — so the caller never mistakes give-up for "everything was tried". + pub untried_routes: Vec, + /// Playwright MCP can only be driven from the agent session, so it is, by + /// construction, an untried route the engine cannot perform. + pub must_invoke_playwright_mcp: bool, +} + +impl FetchResult { + /// Length of the fetched body in bytes (content itself is excluded from + /// serialization — see the `content` field). + pub fn content_length(&self) -> usize { + self.content.len() + } +} diff --git a/engine-src/src/safety.rs b/engine-src/src/safety.rs new file mode 100644 index 0000000..597fe90 --- /dev/null +++ b/engine-src/src/safety.rs @@ -0,0 +1,7 @@ +//! SSRF guard (M1, RFC 0001 §4.7). +//! +//! Validates the initial URL **and every redirect hop** against a +//! private/loopback/link-local/cloud-metadata block-list before any request is +//! made. Redirects are followed manually so each hop is re-checked. +// +// TODO(M1): classify_url(), redirect resolution, IP/CIDR block-list (ipnet). diff --git a/engine-src/src/scheduler.rs b/engine-src/src/scheduler.rs new file mode 100644 index 0000000..4645f2a --- /dev/null +++ b/engine-src/src/scheduler.rs @@ -0,0 +1,9 @@ +//! Diversity scheduler — the heart (M2, RFC 0001 §4.5, §4.6). +//! +//! Materializes the grid (url_transforms × tls_impersonate × referer) across the +//! top-3 detected profiles, orders it for diversity (vary TLS *family* fastest), +//! deprioritizes `avoid` targets without deleting them, and is exhaustive by +//! default (R6). On failure it emits the R6 failure gate: `untried_routes` + +//! `must_invoke_playwright_mcp`. +// +// TODO(M2): build_plan() diversity ordering, run loop, failure-gate reporting. diff --git a/engine-src/src/transport.rs b/engine-src/src/transport.rs new file mode 100644 index 0000000..b695cc7 --- /dev/null +++ b/engine-src/src/transport.rs @@ -0,0 +1,7 @@ +//! Transport: rquest session pool + warmup + cookie bridge (M1, RFC 0001 §4.7). +//! +//! Per-(host, impersonate) session reuse so WAF sensor cookies mature across +//! attempts; root warmup for deep URLs; browser→curl cookie bridge so one +//! expensive Playwright pass converts into cheap impersonated-HTTP throughput. +// +// TODO(M1): rquest client, SessionPool, warmup(), inject_cookies(), request(). diff --git a/engine-src/src/url_transforms.rs b/engine-src/src/url_transforms.rs new file mode 100644 index 0000000..f503e7f --- /dev/null +++ b/engine-src/src/url_transforms.rs @@ -0,0 +1,6 @@ +//! Generic URL transforms for the fetch grid (M2, RFC 0001 §4.5). +//! +//! Domain-agnostic *rules* (`original`, `mobile_subdomain`, `am_prefix`, +//! `drop_www`) — never a site name. Each transform applies or is skipped. +// +// TODO(M2): apply_transform(), iter_transformed() with dedup. diff --git a/engine-src/src/validators.rs b/engine-src/src/validators.rs new file mode 100644 index 0000000..3333ae8 --- /dev/null +++ b/engine-src/src/validators.rs @@ -0,0 +1,8 @@ +//! 4-layer response validation (M1, RFC 0001 §4.2). +//! +//! Layered AND check producing a [`crate::Verdict`]: status semantics → hard +//! challenge markers → size fingerprint → JSON awareness → caller positive +//! proof (CSS selectors) → no-proof heuristics (soft markers, tiny body, +//! unresolved sensor cookie). +// +// TODO(M1): validate(resp, success_selectors, known_bad_sizes) -> Verdict. diff --git a/engine-src/src/waf_detector.rs b/engine-src/src/waf_detector.rs new file mode 100644 index 0000000..30c4f4c --- /dev/null +++ b/engine-src/src/waf_detector.rs @@ -0,0 +1,8 @@ +//! Ranked WAF-product detection (M2, RFC 0001 §4.4). +//! +//! Scores each profile in `engine/waf_profiles.yaml` against the live response +//! (cookies / headers / server / body markers) and returns a ranked list of +//! `(profile_id, confidence)` — never a single verdict (a wrong single guess +//! cascades into a wrong plan). +// +// TODO(M2): load profiles (serde_yaml), detect() ranking, unknown_challenge net. diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..16741ef --- /dev/null +++ b/mise.toml @@ -0,0 +1,28 @@ +# Pinned local toolchain (Skill("standards:dev-tooling")). +# rust — the engine (engine-src/). curl_cffi's Rust analogue (rquest/wreq) is a +# plain crate, so the toolchain is just rust + cargo. +# bun — tooling only: @pleaseai/worktreeinclude in orca.yaml, and (M6) the +# optional npm-distributed wrapper. The engine itself needs no JS. +[tools] +rust = { version = "1.94", components = "rustfmt,clippy", profile = "minimal" } +bun = "latest" + +[tasks.build] +run = "cargo build" + +[tasks.test] +run = "cargo test" + +[tasks.fmt] +run = "cargo fmt --all" + +[tasks.fmt-check] +run = "cargo fmt --all -- --check" + +[tasks.lint] +run = "cargo clippy --all-targets -- -D warnings" + +# One command for the full local gate (mirrors CI). +[tasks.check] +depends = ["fmt-check", "lint"] +run = "cargo test --locked" diff --git a/orca.yaml b/orca.yaml new file mode 100644 index 0000000..6f55038 --- /dev/null +++ b/orca.yaml @@ -0,0 +1,19 @@ +# orca.yaml — setup script run when Orca creates a worktree (committed, team-shared). +# Rust + mise variant. Details: Skill("standards:dev-tooling") -> references/worktree-setup.md +# +# Injected environment variables: +# $ORCA_ROOT_PATH — main checkout (source for copying gitignored local files) +# $ORCA_WORKTREE_PATH — the newly created worktree (copy target + setup cwd) + +scripts: + setup: | + set -e + # 1. Trust the new worktree's mise.toml (trust is per-path). + mise trust --quiet + # 2. Install the pinned toolchain (rust + bun) declared in mise.toml [tools]. + mise install + # 3. Copy gitignored local files (.env, .claude/settings.local.json) into the worktree. + # Run via `mise exec -- bunx` so the mise-installed bun is used regardless of host PATH. + mise exec -- bunx @pleaseai/worktreeinclude "$ORCA_ROOT_PATH" "$ORCA_WORKTREE_PATH" + # 4. Pre-fetch crate dependencies so the first build in the worktree is fast. + mise exec -- cargo fetch diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..f42c8b3 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,2 @@ +edition = "2021" +max_width = 100 diff --git a/skills/adaptive-fetch/SKILL.md b/skills/adaptive-fetch/SKILL.md new file mode 100644 index 0000000..bc039b0 --- /dev/null +++ b/skills/adaptive-fetch/SKILL.md @@ -0,0 +1,28 @@ +--- +name: adaptive-fetch +description: > + Auto-bypass for blocked websites — tries every site-agnostic strategy until one + works. Use when WebFetch returns 402/403/blocked or hits a WAF/CAPTCHA. UNDER + CONSTRUCTION (M0 scaffold): the Rust engine is not wired up yet, so do not rely + on this skill for real fetches. See docs/rfcs/0001-adaptive-fetch.md. +--- + +# adaptive-fetch (under construction) + +> ⚠️ **M0 scaffold.** The engine (`engine-src/`) currently returns an honest +> "not implemented" result. The harness rules (R1–R7), Phase 0 index, and usage +> contract below are being ported from the design and will activate as the +> milestones land. + +The full design — architecture, invariants, and the milestone plan — lives in +[`docs/rfcs/0001-adaptive-fetch.md`](../../docs/rfcs/0001-adaptive-fetch.md). + +## Engine contract (target) + +```bash +adaptive-fetch "" [--selector ""]... [--device auto|desktop|mobile] [--trace] [--json] +# exit 0 = validated success, exit 1 = failure (with untried_routes in --json) +``` + +Until M1+ ships, treat a non-zero exit and `stop_reason="unimplemented"` as +"engine not ready" rather than "site is unreachable".