diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 95310fc..53fa651 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -7,8 +7,8 @@ { "name": "webmcpify", "source": "./", - "description": "Integrate WebMCP into any existing web app — inventory, integrate, verify, heal — without touching unrelated logic or UI.", - "version": "0.4.0" + "description": "WebMCP agent skill for curated core coverage or route-by-route parity — inventory an existing web app, integrate approved tools, then verify and heal them in a real browser.", + "version": "0.5.0" } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 9201500..f614366 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "webmcpify", - "description": "Make any web app agent-ready, verifiably: an end-to-end pipeline that inventories user actions into a reviewable tool manifest, integrates WebMCP tools, then verifies and heals them in a real browser — with zero unrelated changes.", - "version": "0.4.0", + "description": "WebMCP agent skill for curated core coverage or route-by-route parity — inventory an existing web app, integrate approved tools, then verify and heal them in a real browser.", + "version": "0.5.0", "author": { "name": "Jonas Tüchler" } diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index 9201500..f614366 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "webmcpify", - "description": "Make any web app agent-ready, verifiably: an end-to-end pipeline that inventories user actions into a reviewable tool manifest, integrates WebMCP tools, then verifies and heals them in a real browser — with zero unrelated changes.", - "version": "0.4.0", + "description": "WebMCP agent skill for curated core coverage or route-by-route parity — inventory an existing web app, integrate approved tools, then verify and heal them in a real browser.", + "version": "0.5.0", "author": { "name": "Jonas Tüchler" } diff --git a/AGENTS.md b/AGENTS.md index 0ac7de6..6818cdf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,9 +11,12 @@ Quick facts: (loop) → VERIFY (loop) → HEAL (loop) → AUDIT + report. Invocation modes (`inventory` / `integrate` / `verify` / `status` / `full`) run subsets — see SKILL.md §Invocation modes. -- All state persists in `.webmcpify/manifest.json` (Manifest v2) in the target +- All state persists in `.webmcpify/manifest.json` (Manifest v4) in the target repo — resume from it if it exists. Terminal tool statuses: `verified`, `skipped`, `rejected`. +- Coverage is explicit: `curated` produces a reviewed route→tool map; `parity` + produces a per-route interactive-element census. Neither mode may infer coverage + from a raw tool count. - `skills/webmcpify/templates/` contains everything you vendor into target projects (runtime TS + JS, ambient types, Playwright spec) — the skill directory is self-contained; nothing outside it is needed at run time. diff --git a/CHANGELOG.md b/CHANGELOG.md index c5a7521..bbcec5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ reconstruct them from git history. ## [Unreleased] +- Added an explicit `curated | parity` coverage choice, policy-backed inventory + verdicts, and a required route→tool coverage map (an element census for parity). +- Made secure context, verification origin, backend origins and CORS assumptions + DETECT gates instead of late verification surprises. +- Guarded imperative tools against bare `null`/`undefined` results and documented + deferred route changes after a structured result. +- Classified failures before retry counting, reset counters after contract changes, + and made skips state the impossibility class and evidence. +- Hardened headed-Chrome verification around environment-provided origins and + dedicated profiles, and added dated ChatGPT Site tools guidance. + ## [0.4.0] — 2026-08-17 - Migrated persistent runs to Manifest v3 with explicit auth fixtures, client vs. diff --git a/README.md b/README.md index 67e22f2..1dcd732 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ 🌐 **[webmcpify.at](https://webmcpify.at)** — the site itself is webmcpified: open it with a WebMCP-enabled agent and call its tools. -webmcpify is an agent skill that integrates [WebMCP](https://webmachinelearning.github.io/webmcp/) +webmcpify is a WebMCP agent skill for **curated core coverage or route-by-route +parity**. It integrates [WebMCP](https://webmachinelearning.github.io/webmcp/) (`document.modelContext` — a proposed web standard incubated in the W3C Web Machine Learning Community Group, currently in Chrome origin trial) into an **existing** web application — from a static landing page to a large multi-tenant SaaS — end to end: @@ -14,11 +15,12 @@ DETECT ─▶ INVENTORY ─▶ [you approve the tool manifest] ─▶ INTEGRATE loop loop loop loop ``` -Your coding agent investigates the codebase, proposes a **tool manifest** (every -user action worth exposing, with names, schemas, examples, and a read-only/mutating -classification), and after your approval integrates the tools, **exercises each one -in a real browser**, and heals failures — escalating honestly what it can't fix — -while keeping unrelated logic and UI untouched. +Your coding agent investigates the codebase, asks whether you want a curated set of +high-value actions or an auditable per-route interaction census, then proposes a +**tool manifest** with names, schemas, examples, coverage reasons, and a read-only/ +mutating classification. After your approval it integrates the tools, **exercises +each one in a real browser**, and heals failures—while keeping unrelated logic and +UI untouched. ## See native Chrome verification @@ -76,6 +78,7 @@ Open your agent in the target repo and pick your scope: /webmcpify integrate # integrate the approved manifest /webmcpify verify # verify + heal what's integrated /webmcpify status # where are we? what's next? +/webmcpify full parity # census every interactive element on every authenticated route ``` (or in plain words: *"webmcpify this app"*, *"map what tools this app could expose"*) @@ -95,8 +98,13 @@ Every phase is a **loop over persistent state**, not a one-shot pass: deep-reads one area per iteration — a 500-file SaaS is processed area by area, never in one context-busting sweep. Sub-agent fan-out writes per-area shard files; a single coordinator merges them (no write races). -- **Tool budgets** keep SaaS toolsets usable: priority waves, an overlap rule +- **Coverage is explicit:** `curated` produces a reviewed route→tool map for core + actions; `parity` produces a per-route element census where every interaction is + mapped to a tool or a written reason. A tool count alone is never called 100%. +- **Tool budgets** keep curated SaaS toolsets usable: priority waves, an overlap rule (no two tools matching the same request), and role/tenant coverage tracking. + Parity uses route-scoped registration and reports client-capacity gaps instead of + claiming an unmeasured universal per-page limit. - **Integrate** works in small batches (one area or ≤5 tools), each independently built and typechecked — committed per batch only if you opted in. - **Verify/Heal** iterate per tool with attempt caps and honest escalation @@ -108,13 +116,17 @@ Every phase is a **loop over persistent state**, not a one-shot pass: - **Unrelated logic and UI stay untouched** — every diff hunk traces to a manifest entry; a final audit against the recorded baseline commit enforces it, and files that were already dirty when the run started are never modified or reverted. -- **Read-only first** — mutating tools require your explicit per-tool approval; - destructive/payment actions are never exposed. +- **Read-only first** — server mutations require your explicit per-tool approval. + Auth, signup, billing, payment and credential-returning tools stay excluded; + irreversible delete actions can only open the app's existing confirmation UI. - **Server stays the trust boundary** — tools only call code paths your UI already uses; no new endpoints, no bypasses. - **Spec-shaped, zero dependencies** — a small MIT runtime is vendored into your repo (no npm dependency), everything feature-detected: your app is **behaviorally unchanged** in browsers without WebMCP. +- **No ambiguous imperative results** — the runtime guards accidental bare + `null`/`undefined`, and route-changing tools return a structured result before + deferring navigation and route-scope disposal. - **Exercised, not assumed** — every tool is enumerated and executed in real Chrome, asserting on both the tool result and the resulting UI state, from examples recorded in the manifest. That includes mutating declarative forms, @@ -149,6 +161,9 @@ enumeration/execution surface, and treats Google's live source of current best practices at integration time. Release-by-release spec adaptations are recorded in the [changelog](CHANGELOG.md). +ChatGPT's separate, model/account-gated client surface is documented as +[Site tools](skills/webmcpify/references/client.md), with dated availability facts +and a troubleshooting order. ## Related projects diff --git a/gemini-extension.json b/gemini-extension.json index d8dc1e3..f614366 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,7 +1,7 @@ { "name": "webmcpify", - "description": "Integrate WebMCP into any existing web app — inventory, integrate, verify, heal — without touching unrelated logic or UI.", - "version": "0.4.0", + "description": "WebMCP agent skill for curated core coverage or route-by-route parity — inventory an existing web app, integrate approved tools, then verify and heal them in a real browser.", + "version": "0.5.0", "author": { "name": "Jonas Tüchler" } diff --git a/package-lock.json b/package-lock.json index 7c0d519..444b9b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "webmcpify", - "version": "0.4.0", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "webmcpify", - "version": "0.4.0", + "version": "0.5.0", "license": "MIT", "devDependencies": { "@playwright/test": "^1.54.0", diff --git a/package.json b/package.json index fde27d6..2f1cce6 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "webmcpify", "displayName": "webmcpify", - "description": "WebMCP agent skill — make any web app agent-ready, verifiably: inventory, integrate, verify in a real browser, heal.", + "description": "WebMCP agent skill for curated core coverage or route-by-route parity — inventory an existing web app, integrate approved tools, then verify and heal them in a real browser.", "type": "module", "license": "MIT", - "version": "0.4.0", + "version": "0.5.0", "publisher": "TueJon", "repository": { "url": "https://github.com/TueJon/webmcpify" diff --git a/release/v0.5.0-directory-update.md b/release/v0.5.0-directory-update.md new file mode 100644 index 0000000..9edb079 --- /dev/null +++ b/release/v0.5.0-directory-update.md @@ -0,0 +1,37 @@ +# webmcpify v0.5.0 — directory update draft + +Status: copy and refresh plan only. Do not submit, post, publish, buy placement or +change ads without a separate approval. + +## Short directory copy + +webmcpify is an open-source WebMCP agent skill for existing web apps. Choose curated +core coverage or route-by-route parity; it inventories a reviewable tool manifest, +integrates only approved tools through the app's existing UI paths, and verifies and +heals them in real headed Chrome. v0.5.0 adds secure-origin/CORS gates, +policy-backed coverage evidence, structured results for route-changing tools and +failure-aware retries. + +Canonical source: + + +## Refresh after v0.5.0 is actually published + +- **GitHub:** publish the prepared release notes and verify the repository About/ + README description carries `curated | parity`. +- **skills.sh and SkillsMP:** expect source-driven refresh; verify the displayed + version/description after their next crawl because skills.sh previously served a + stale cached revision. Do not create duplicates. +- **Awesome Skills:** request a recrawl/correction of the live canonical listing; + its provider copy called the project an “Antigravity agent skill”. +- **mcpservers.org:** correct the live listing from “MCP Server” to “WebMCP agent + skill” and replace its description with the copy above. +- **MCP.Directory:** the accepted review had no live listing as of 2026-07-23. + Re-check exact-name URLs; do not resubmit while the existing review is pending. +- **agentskills.in / agentskill.sh / agenticskills.io:** re-check provider issue/ + queue health before any retry; prior submissions failed on provider-side 500, + GitHub API 403, storage quota or an unconfigured review queue. +- **OpenAgentSkill:** re-check the live star threshold before submitting. + +The weekly public-reach automation remains the measurement owner; do not add a +second reach checker. diff --git a/release/v0.5.0.md b/release/v0.5.0.md new file mode 100644 index 0000000..e5d7350 --- /dev/null +++ b/release/v0.5.0.md @@ -0,0 +1,37 @@ +# webmcpify v0.5.0 — draft release notes + +Status: prepared, not published. Re-check the dated client claims and run the +release checklist before creating a GitHub release or directory update. + +webmcpify now asks for a coverage target up front: + +- `curated` maps a reviewed set of high-value actions and records why other + interaction classes were deferred. +- `parity` inventories every interactive element on every authenticated route and + maps each one to a WebMCP tool or a written reason. The route census—not a tool + count—is the audit evidence. + +The rest of the pipeline is stricter where real integrations failed: + +- DETECT refuses to inventory on an insecure verification origin and records + backend-origin/CORS assumptions before tool work begins. +- Inventory verdicts carry the policy that produced them; widening a gate reopens + affected areas automatically. +- Identity, tenancy/billing, credentials and destructive-delete gates use separate + vocabulary, so creating ordinary product objects is no longer mistaken for + account creation. +- Imperative tools cannot return bare `null`/`undefined`; route-changing tools + return a structured result before deferring navigation and disposal. +- HEAL classifies failures before counting independent retries and resets the + counter after an approved contract revision. +- The verification template requires a secure env-provided origin, a dedicated + Chrome profile and a real headed browser environment. + +The skill also documents ChatGPT's **Site tools** surface separately from Chrome +developer verification. Availability details are dated 2026-08-27 and link to the +official OpenAI documentation because model/workspace support can change. + +Compatibility note: no public source or measured client run establishes a universal +safe tool count per page. Parity is an auditable coverage target, not a promise that +every client will accept an arbitrarily large route toolset; client-capacity gaps are +reported honestly. diff --git a/skill.json b/skill.json index e7bd3b6..f322f83 100644 --- a/skill.json +++ b/skill.json @@ -1,7 +1,7 @@ { "name": "webmcpify", - "version": "0.4.0", - "description": "WebMCP agent skill — make any web app agent-ready, verifiably: inventory, integrate, verify in a real browser, heal.", + "version": "0.5.0", + "description": "WebMCP agent skill for curated core coverage or route-by-route parity — inventory an existing web app, integrate approved tools, then verify and heal them in a real browser.", "license": "MIT", "author": { "name": "Jonas Tüchler", @@ -23,7 +23,7 @@ { "id": "webmcpify", "path": "skills/webmcpify/SKILL.md", - "description": "End-to-end WebMCP integration pipeline: propose a tool manifest, integrate, verify every tool in a real browser, heal failures — zero unrelated changes." + "description": "Choose curated core coverage or route-by-route parity, inventory an existing web app, integrate approved WebMCP tools, then verify and heal them in a real browser." } ], "agents": ["claude-code", "codex", "cursor", "opencode", "copilot", "gemini-cli"], diff --git a/skills/webmcpify/SKILL.md b/skills/webmcpify/SKILL.md index c45143a..cc51218 100644 --- a/skills/webmcpify/SKILL.md +++ b/skills/webmcpify/SKILL.md @@ -1,6 +1,6 @@ --- name: webmcpify -description: WebMCP agent skill that makes a web app agent-ready — propose a WebMCP tool manifest, integrate, verify in a real browser, heal; unrelated code stays untouched. Use for "webmcpify", "add WebMCP", or "expose app actions to AI agents". +description: WebMCP agent skill for curated core coverage or route-by-route parity — inventory an existing web app, integrate approved tools, then verify and heal them in a real browser. Use for "webmcpify", "add WebMCP", or "expose app actions to AI agents". argument-hint: "[inventory|integrate|verify|status|full] [scope notes]" license: MIT tags: @@ -59,8 +59,11 @@ Any other text is scoping guidance (e.g. "only the checkout area", "read-only to `"client"` (browser-local only: prefs, localStorage), or `"server"` (data leaves the browser). Server-mutating tools require explicit **per-tool** human approval recorded in the manifest; client-mutating tools may be approved as a - batch at the gate. Never expose destructive, irreversible, or payment actions - in a first integration. + batch at the gate. The scope floor is precise: no auth/login/session/password/ + MFA/SSO; no signup/registration/payment/billing/subscription; no tool returning + a credential, token, key, JWT, signed URL, or cookie; and no irreversible delete + except by opening the app's own confirmation UI. Creating or changing ordinary + product objects is in scope and must not be mistaken for account creation. 3. **The server is the only trust boundary.** A tool's `execute()` may only call code paths the UI already uses (same endpoints, same validation, same auth). Never create new endpoints, never bypass existing checks, never put secrets in tools. @@ -105,10 +108,22 @@ shards) before redispatching any sub-agents. Then continue at `pipeline.phase`, the first `pending` area, or the first tool whose status is not terminal. Terminal statuses: `verified`, `skipped`, `rejected`. +An inventory verdict is reusable only under the policy that produced it. Before +honouring an `inventoried` area, compare its `policyFingerprint` with +`pipeline.inventoryPolicy.fingerprint`. When a named gate is widened or removed, +mechanically reset every area whose `exclusions` cite that gate to `pending`, clear +its derived route-coverage entries and unapproved discovered tools, and record the +invalidation in `log`. A zero-candidate area without usable exclusion provenance is +also reset. Editing `status` alone is never an invalidation. Approved or integrated +tools affected by a later policy change return to the human gate. + **Phase transitions** (make the atomic manifest write the moment the condition holds): -- `detect → inventory`: `app` recorded, `baselineSha`/`baselineDirty` captured. -- `inventory → gate`: no area `pending`, completeness pass has run. +- `detect → inventory`: `app` recorded, `app.secureContext === true` at the recorded + `app.verificationOrigin`, backend/CORS assumptions captured, `coverageTarget` and + inventory policy recorded, and `baselineSha`/`baselineDirty` captured. +- `inventory → gate`: no area `pending`, route coverage is recorded, and the + target-specific completeness pass has run. - `gate → integrate`: every `discovered` tool is `approved`/`rejected`, and `commitPolicy` + `commitWebmcpifyDir` are set. - `integrate → verify`: no `approved` tools remain (each `integrated` or terminal), @@ -118,13 +133,17 @@ Terminal statuses: `verified`, `skipped`, `rejected`. - `heal → audit`: no tool `failed` and post-heal full re-verify passed. - `audit → done`: every hunk mapped-or-flagged, `report.md` finalized. -Manifest schema (Webmcpify Manifest v3): +Manifest schema (Webmcpify Manifest v4): ```jsonc { - "webmcpify": 3, + "webmcpify": 4, "app": { "stack": "react-vite", "typescript": true, "entry": "src/main.tsx", - "baseUrl": "http://localhost:5173", "startCommand": "npm run dev", + "baseUrl": "https://app.example.test", "startCommand": "npm run dev", + "verificationOrigin": "https://app.example.test", + "secureContext": true, + "backendOrigins": ["http://localhost:3000"], + "corsAllowlist": ["https://app.example.test"], "authFixtures": { // how verify OBTAINS each session "member": { "obtain": "npm run seed:test-user, then sign in at /login", "account": "member@example.test", @@ -132,6 +151,17 @@ Manifest schema (Webmcpify Manifest v3): } }, "pipeline": { "phase": "inventory", // detect|inventory|gate|integrate|verify|heal|audit|done — transition rules above + "coverageTarget": "parity", // REQUIRED before inventory: "curated" | "parity"; never silently default + "inventoryPolicy": { + "revision": 1, + "fingerprint": "sha256:", + "gates": { + "identity": "exclude auth/login/session/password/MFA/SSO", + "tenancy_billing": "exclude signup/registration/payment/billing/subscription", + "credentials": "exclude tools returning credentials/tokens/keys/JWTs/signed URLs/cookies", + "irreversible_delete": "only open the app's own confirmation UI" + } + }, "setup": { // PATHS created/modified per one-time setup step ([] = not done yet) "runtimeVendored": ["src/webmcp/webmcpify.ts", "src/webmcp/webmcp.d.ts"], "harnessInstalled": [".webmcpify/webmcp.spec.ts"], @@ -152,7 +182,19 @@ Manifest schema (Webmcpify Manifest v3): "blockers": [] // e.g. "app won't start locally: needs $API_KEY" — surfaced at the gate }, "areas": [ - { "id": "checkout", "paths": ["src/features/checkout/"], "status": "pending" } // pending|inventoried + { "id": "tickets", "paths": ["src/features/tickets/"], "routes": ["/projects/:id/tickets"], + "status": "inventoried", // pending|inventoried + "policyFingerprint": "sha256:", + "exclusions": [{ "gate": "identity", "reason": "login form" }] } + ], + "routeCoverage": [ + { "route": "/projects/:id/tickets", "area": "tickets", "auth": ["role:member"], + "interactions": [ + { "element": "New ticket button", "source": "src/features/tickets/List.tsx:42", + "tool": "create_ticket", "reason": null }, + { "element": "Account menu", "source": "src/layout/AccountMenu.tsx:18", + "tool": null, "reason": "identity gate: logout/session action" } + ] } ], "tools": [ { @@ -171,14 +213,16 @@ Manifest schema (Webmcpify Manifest v3): // invalid: null ONLY for readOnlyHint tools with no/empty params — // verify then asserts dual-outcome: rejects OR resolves with no side effect "expect": { "result": "created", "navigation": null, "ui": "new row appears in the ticket list" }, - // exactly one of result|navigation: result = substring of the resolved string; - // navigation = destination URL/pattern when executeTool resolves null (it navigated) + // result = substring of the serialized structured result; + // navigation = destination URL/pattern after a declarative submit or deferred imperative route action "cleanup": "delete the created ticket via the UI's own delete path (test data only)", // required for mutating:"server", recommended for "client" "status": "discovered", // discovered|approved|rejected*|integrated|verified*|failed|skipped* (* = terminal) "approval": null, // server-mutating tools, once approved: { "note": "...", "at": "2026-07-12", // "productionSideEffect": null } — set only when verification unavoidably // causes a real production effect (see VERIFY: production side-effect policy) - "attempts": 0, // heal-fix cycles; the triggering verify failure is attempt 0 + "contractRevision": 1, + "failure": null, // on failure: { "class": "contract|implementation|environment|external-policy|flaky|client-capacity", "signature": "...", "contractRevision": 1 } + "attempts": 0, // independent retries of this failure signature under this contract revision "batchCommit": null, // sha under commit-per-batch — lands in the manifest one commit LATER "notes": "" } @@ -187,15 +231,37 @@ Manifest schema (Webmcpify Manifest v3): } ``` -**v2→v3 migration:** resuming a `"webmcpify": 2` manifest migrates in place on -first write — `auth` string → array; `setup` booleans → path arrays (`false` → -`[]`; `true` → recover paths from git/`log`, else `null` = done-but-unrecorded, -audit treats those files flag-only); `mutating: true` → `"server"`; add -`annotations` (defaults from the inventory table), `blockers: []`, -`commitWebmcpifyDir: null`, `expect.navigation: null`; then bump to 3. +**v2/v3→v4 migration:** first perform the existing v2→v3 conversions (`auth` +string → array; setup booleans → path arrays; `mutating: true` → `"server"`; +annotations/blockers/commitWebmcpifyDir/navigation defaults). Then require a +`coverageTarget` choice, capture the current inventory policy, add the origin/CORS +fields, `routeCoverage: []`, and tool `contractRevision`/`failure` fields. Existing +inventoried areas get `policyFingerprint: null` and are reset to `pending`; their +old zero-candidate verdicts are not trusted. Bump to 4 only after persisting that +invalidation. ## Phase 0 — DETECT +**The first browser gate is secure context.** Read only enough startup config to +boot the app at a candidate verification origin, open it in headed Chrome, and +evaluate `window.isSecureContext`. Record the exact origin and result. HTTPS and +loopback origins can qualify; a plain-HTTP non-loopback origin does not. The +`WebMCPTesting` feature flag does not waive this gate. If false, record the blocker +and refuse to enter INVENTORY. + +Capture the app's absolute backend-origin assumptions and exact CORS allow-list, +then choose a verification origin compatible with both. If the page never boots, +save console errors and failed requests before changing any tool code: an absolute +localhost backend URL or port-pinned CORS rule is an environment failure, not a +WebMCP integration failure. + +Choose `pipeline.coverageTarget` explicitly before inventory: `curated` maps a +reviewed set of high-value actions; `parity` performs an exhaustive interactive- +element census per authenticated route. There is no measured universal client +tool-count ceiling, so do not promise that a large parity toolset is safe merely +because registration is route-scoped; verification must enumerate each route in +the target clients actually available. + Identify stack, build + dev-server commands, TypeScript or not, auth model (including how verify obtains each test session → `app.authFixtures`), test setup, and how the app starts locally; record under `app`. Record the git baseline: @@ -221,12 +287,18 @@ verification will be blocked and this must be surfaced at the gate. Details: `"inventoried"`, write the manifest, repeat. - **Sub-agent fan-out:** sub-agents never write `manifest.json`. Each writes only its own `areas/.tools.json` shard — schema - `{ "webmcpifyShard": 3, "area": "", "tools": [ /* full v3 tool entries */ ] }`, + `{ "webmcpifyShard": 4, "area": "", "tools": [ /* full v4 tool entries */ ] }`, written atomically (tmp + rename). You (the coordinator) merge shards into the manifest sequentially, then delete them; on resume, merge existing shards FIRST before redispatching (Resume rule). -3. **Exit:** no `pending` areas remain, plus one completeness pass — walk the app's - navigation and ask "is any visible user action missing?" +3. **Coverage output:** populate `routeCoverage` in both modes. `curated` maps the + selected tools and records why deliberately omitted interaction classes were + left out. `parity` inventories every interactive element on every authenticated + route and maps each to a tool or a written reason. Pay explicit attention to + deletes, drag/drop ordering, bulk and multi-select, table sort/columns/pagination/ + saved filters, invitations, membership and permission writes, settings toggles, + and canvas/viewer controls. +4. **Exit:** no `pending` areas remain, plus the coverage-target completeness pass. ## GATE — manifest approval (the one main checkpoint) @@ -248,6 +320,11 @@ exchange where possible: Origin-allow-listed endpoint), get that approved HERE and record it in the tool's `approval.productionSideEffect` — see VERIFY. +If the human changes a scope gate, update `pipeline.inventoryPolicy`, compute its +new fingerprint, apply the mandatory invalidation rule before presenting the +manifest again, and show which areas were reopened. Never carry forward a verdict +without its producing policy. + Apply `references/security.md` to every mutating tool **before** presenting. ## Phase 2 — INTEGRATE (loop) @@ -293,7 +370,7 @@ hunk — leave it alone). ## Phase 3 — VERIFY (loop) Set up once from `templates/webmcp.spec.ts` per `references/verify.md` (real headed -Chrome; production `getTools()`/`executeTool()` surface with legacy fallback probe). +Chrome; current production `document.modelContext.getTools()`/`executeTool()` surface). Then loop over every `integrated` tool, using its manifest `route`, `auth`, `examples`, `expect`, and `annotations` fields: @@ -303,8 +380,9 @@ Then loop over every `integrated` tool, using its manifest `route`, `auth`, - execute the valid example (mutating tools: dev/test data only, then run `cleanup`) and one invalid example (`invalid: null` zero-param read tools: dual-outcome assertion — see `references/verify.md`); -- assert on the returned result **and** the resulting UI state per `expect` - (a UI **delta**, or `expect.navigation` when execution resolves `null`). +- assert on the returned structured result **and** the resulting UI state per + `expect`; for route actions, assert `expect.navigation` after the deferred + app navigation. A bare imperative `null`/`undefined` result is a failure. Pass → `"verified"`. Fail → `"failed"` + failure note. Role-scoped tools: run the loop once per role listed in `auth`, signing in via the matching @@ -323,10 +401,14 @@ While any tool is `"failed"`: diagnose via `references/heal.md`, fix **only** th tool's integration — **implementation-only** fixes; if the fix would change the approved contract (schema, description, `mutating` class, `annotations`, `expect`), go back to the gate for re-approval instead of silently changing the -manifest. The triggering verify failure is attempt 0; increment `attempts` per -fix cycle and re-verify. At `attempts` = 3 → `"skipped"` with a clear blocker -note (an explicit escalation to the human, not a silent drop). Never widen the -diff or fake a pass. After healing, re-run verification once for **all** tools +manifest. Classify the failure before counting it. `attempts` counts only +independent retries of the same failure signature under the same contract revision; +environment and shared contract failures do not burn one attempt on every affected +tool. When the contract changes after re-approval, increment `contractRevision`, +clear `failure`, and reset `attempts`. At three independent failed retries of an +unchanged implementation/flaky signature, `"skipped"` requires the failure class, +signature, and evidence that makes the tool impossible under the current contract. +Never widen the diff or fake a pass. After healing, re-run verification once for **all** tools with status `integrated` or `verified` (healing one tool can break another — scope collisions). @@ -345,7 +427,8 @@ scope collisions). manifest `source` fields, `pipeline.setup` paths, and `pipeline.discovery.paths` (setup entries recorded as `null` by the v2→v3 migration: fall back to flag-only for those files). -2. Finalize `.webmcpify/report.md`: tool coverage per area, skipped/rejected tools +2. Finalize `.webmcpify/report.md`: the enumerated route→tool map, coverage target, + tool coverage per area, skipped/rejected tools with reasons, security notes (which mutating tools exist, what guards them, any recorded production side effects), how to test manually (flag, DevTools WebMCP pane, inspector extension), and every blocker that needs a human. @@ -361,3 +444,4 @@ scope collisions). - `references/discovery.md` — optional off-page discovery (manifest, `rel="webmcp"`, `llms.txt`) + how to read third-party audit scores - `references/security.md` — the security checklist (apply before the gate and at audit) +- `references/client.md` — dated ChatGPT Site tools availability and troubleshooting diff --git a/skills/webmcpify/references/client.md b/skills/webmcpify/references/client.md new file mode 100644 index 0000000..7a7c798 --- /dev/null +++ b/skills/webmcpify/references/client.md @@ -0,0 +1,47 @@ +# ChatGPT client reality — Site tools + +Checked 2026-08-27 against the official OpenAI documentation: +. Re-check that page before publishing or +relying on model/workspace availability; this UI is moving independently of the +WebMCP draft and Chrome implementation. + +## What is documented + +- ChatGPT calls WebMCP integrations **Site tools**. In the desktop app's built-in + browser, open **Site tools** in the address bar and then **Available site tools** + to inspect the page's registrations. This is the fastest client-side ground + truth for the currently open page. +- Site tools are available to ChatGPT Work and Codex with GPT-5.6 Sol or GPT-5.6 + Terra. GPT-5.6 Luna currently has WebMCP disabled. The desktop app must be + current, and availability still depends on rollout and the current page. +- The official page says Site tools are unavailable in Enterprise and Edu + workspaces. +- Tools belong to the page that registered them. Closing or navigating away can + make them unavailable; a client that tears down the page between turns cannot + call that old registration. +- Each invocation in the built-in browser receives a safety review. Normal access + and confirmation policies still apply. + +Date every public summary and link the official page. Do not publish model names +from memory. + +## Dated observations, not promises + +One session on 2026-08-26/27 observed plain Chat handing Site-tools work to Work +mode and collapsing the browser pane, while a Work workspace offered Site tools. +Those observations are account/UI-specific and do not override the official +availability statement. Verify them in the exact account before troubleshooting a +customer report or publishing a claim. + +## Troubleshooting order + +1. In the built-in browser, keep the page open and inspect **Site tools → Available + site tools**. +2. Confirm the desktop app is current and the selected model is one the official + page currently lists. +3. Confirm the account/workspace is eligible; absence on the wrong model can look + exactly like a broken integration because the assistant falls back to ordinary + browser interaction. +4. For the website itself, confirm a secure context and live registration. Use the + headed-Chrome harness in `verify.md` for developer proof; ChatGPT availability + and Chrome harness success are separate checks. diff --git a/skills/webmcpify/references/heal.md b/skills/webmcpify/references/heal.md index 9b2e5fd..6db3434 100644 --- a/skills/webmcpify/references/heal.md +++ b/skills/webmcpify/references/heal.md @@ -1,12 +1,25 @@ # Heal — failure taxonomy → fixes -Work one failed tool at a time. Re-verify after each fix. The triggering verify -failure counts as attempt 0; each fix cycle increments `attempts`. At `attempts` -= 3 → mark `skipped` with a blocker note (this is an explicit escalation to the -human in the final report, not a silent drop) and move on. **Never** widen the -diff, disable a check, or fake a return value to force a pass. **Mutating -tools:** run the manifest `cleanup` between attempts — retrying a mutation -without cleanup duplicates data. +Classify before counting. Persist `failure.class`, `failure.signature`, and the +current `contractRevision` before a retry: + +- `contract` — schema/result/lifecycle rule shared by a class of tools; +- `environment` — secure origin, browser/profile/display, app boot, backend/CORS; +- `implementation` — this tool's app wiring is wrong; +- `external-policy` — a real upstream or production-origin restriction; +- `flaky` — the same unchanged path produced independent outcomes; +- `client-capacity` — the target client cannot enumerate/select the intended set. + +`attempts` counts only independent retries of the same failure signature under the +same approved contract revision. A shared contract or environment bug is one root +cause, not one failed attempt per tool: fix it, increment `contractRevision` when +the human-approved contract changed, then clear `failure` and reset `attempts` for +every affected tool. At three independent failed implementation/flaky retries, +`skipped` must name the class, signature, and evidence that makes the tool +impossible under the current contract. A raw count is never sufficient. **Never** +widen the diff, disable a check, or fake a return value to force a pass. **Mutating +tools:** run the manifest `cleanup` between retries — retrying a mutation without +cleanup duplicates data. **Heal failures, not scores.** A low score from a third-party WebMCP inspector/checker is not itself a failure: those lists mix spec features with @@ -26,14 +39,15 @@ gate as a mini re-approval — never silently edit the manifest to match the cod | Symptom | Likely cause | Fix | |---|---|---| -| Tool absent from enumeration | **Registration is async** — the test asserted before `registerTool()` settled; or registration never ran (bootstrap not reached, view not mounted) or wrong Chrome build/flags | FIRST make the test poll (`waitForTool`) or await `toolchange` — only if it still fails, trace the registration call; confirm `isWebMCPAvailable()` in the test env; current Chrome + `--enable-features=WebMCP,WebMCPTesting` | +| Tool absent from enumeration | Insecure verification origin; **registration is async**; bootstrap/view not reached; reused Chrome profile; wrong headed Chrome/flags | FIRST assert `window.isSecureContext` and `document.modelContext`, then poll (`waitForTool`) or await `toolchange`; only then trace registration. `WebMCPTesting` does not waive secure context | +| Page never boots | Absolute backend URL points at an absent localhost service, or CORS allow-list pins another exact origin/port | Capture console + failed requests; correct the verification origin/backend environment before changing tool code. Classify `environment` | | Whole scope absent | A registration in the batch rejected (duplicate name, invalid schema, policy) — the runtime rolls back the entire scope | Check console for the `onError` report; fix the offending tool contract | | Tool absent after route change | Scope disposed by navigation (over-scoping) | Move to static app-level registration unless genuinely view-bound | | Declarative tool missing | `toolname` typo, frame without `allow="tools"`, or page sends `Origin-Agent-Cluster: ?0` | Fix attribute; check Permissions-Policy `tools` and origin-keying headers | | Schema mismatch (declarative) | Control lacks `name`, description not resolvable, unsupported control type in this build | Add `name`/`toolparamdescription`/`label[for]`; unsupported controls → switch that form to imperative | | Schema mismatch (imperative) | Manifest and code drifted | Make code match the approved manifest; if the manifest was wrong, that's a contract change — take it back to the gate for re-approval (see above), never silently update it | | Assertion compares object to string | Enumerated `inputSchema` is a stringified JSON Schema | `JSON.parse` before comparing (see `verify.md`) | -| `executeTool` returns `null` unexpectedly | The execution navigated (normal for submit-navigating declarative forms) | Assert on the post-navigation page instead of the return value | +| `executeTool` returns `null` unexpectedly | A declarative submission navigated; or an imperative handler returned bare `null` | Declarative navigation: assert the destination. Imperative: return a structured result and defer route-changing UI/disposal one event-loop task; the runtime guard converts accidental absence into a structured error | | `executeTool` rejects | Schema violation or declarative-validation failure — rejection IS the failure signal for these | For invalid-input tests on declarative tools, assert rejection, not an `"ERROR:"` string | | Mutating declarative execution hangs until timeout | Chrome fills the form, then **pauses the execution awaiting a real submit interaction** — awaiting `executeTool` alone deadlocks | Use the concurrent pattern in the spec template: start `executeTool` unawaited → wait for the agent-filled value → click submit → await. **NEVER heal by adding `toolautosubmit`** (ground rule 5) | | Backend rejects the harness with 403/CORS despite correct auth | The endpoint **allow-lists the production `Origin`** (mailers, form gateways) — the localhost harness origin is refused before the tool logic runs, and no local fix exists | Verify the live path with the env-gated server-side replay (§Origin-allow-listed endpoints below), only with the production side-effect approval recorded in `approval.productionSideEffect` (see §Origin-allow-listed endpoints below); without it, mark the live path `skipped` with a blocker note | @@ -41,7 +55,7 @@ gate as a mini re-approval — never silently edit the manifest to match the cod | Returns success but UI unchanged | `execute()` bypassed the real UI path (parallel implementation) | Rewrite to call the same handler/store action/endpoint the UI uses | | Invalid input resolves successfully (imperative) | Missing in-code validation | Validate strictly in code; return `"ERROR: "` | | Fetch-submitted form: agent gets nothing | `preventDefault()` without `respondWith()` | Add the `e.agentInvoked → e.respondWith(promise)` bridge | -| Works manually, fails in Playwright | Headless, missing flags, or profile without the flag | Headed + flags; persistent context; `xvfb-run` in CI | +| Works manually, fails in Playwright | Headless, missing flags/display/Chrome, or shared profile whose existing instance ignored the flag | Provision current Chrome + display; use headed persistent context with a dedicated `WEBMCP_PROFILE_DIR` | | 401/403 from `execute()` in test | Tool registered outside the authenticated scope, or test session lacks the role in the manifest `auth` field | Role-scope the registration; sign in with the recorded fixture | | Flaky: passes alone, fails in suite | Shared state between tool executions | Isolate test data per tool run (use `cleanup`); don't reorder tests to hide it | diff --git a/skills/webmcpify/references/integrate.md b/skills/webmcpify/references/integrate.md index 4428e7c..eb6ee81 100644 --- a/skills/webmcpify/references/integrate.md +++ b/skills/webmcpify/references/integrate.md @@ -106,8 +106,33 @@ Key rules: awaits the real work, then fires the completion event with the outcome payload (`{ ok, message | error }`) — full contract and component example in `runtime.md`. A canned success before the work finishes is a false green. -- Return short strings; errors as `"ERROR: "` so the model can - self-correct. Cap outputs ~1.5k chars. +- **Never return bare `null` or `undefined` from an imperative tool.** Current + Chrome builds may serialize it ambiguously or destroy the caller's execution + context even when the UI action succeeded. Return a JSON-safe structured object + such as `{ opened: true, surface: "project-create", route: "/projects/new", + prefilled: false }`; short strings remain valid for simple settled reads. Errors + may use `{ ok: false, error: "what failed and how to fix it" }` or the runtime's + `"ERROR: ..."` convention. Cap serialized outputs around 1.5k characters. +- **Route-changing imperative tools use deferred navigation.** Validate inputs and + construct the structured result first; schedule the app's existing navigation + handler and route-scope disposal together in one later event-loop task, then + return the result immediately. This lets `executeTool()` receive an unambiguous + result before the page destroys its context: + + ```ts + execute: async (input) => { + const route = validateProjectDraft(input); // throws/returns an error before acting + const result = { opened: true, surface: 'project-create', route, prefilled: false }; + setTimeout(() => { + openProjectCreation(route); // the same path the visible UI uses + disposeRouteTools(); + }, 0); + return result; + } + ``` + + `null` remains a browser-owned outcome for a navigating **declarative form**; + application `execute()` handlers must not manufacture it. - Validate strictly in code, loosely in schema — and keep **parity with the form's native HTML constraints**: when a tool wraps a form, probe the real constraints on a detached clone instead of re-implementing them — diff --git a/skills/webmcpify/references/inventory.md b/skills/webmcpify/references/inventory.md index bc35972..1a2cd7c 100644 --- a/skills/webmcpify/references/inventory.md +++ b/skills/webmcpify/references/inventory.md @@ -4,20 +4,32 @@ Establish, in this order: -1. **Stack**: `package.json` deps (react/vue/@angular/next/astro/eleventy…) or the +1. **Secure verification origin (hard gate)**: read only enough startup config to + boot the app, then open the candidate origin in headed Chrome and evaluate + `window.isSecureContext`. Record `app.verificationOrigin` and + `app.secureContext`. HTTPS and loopback origins can qualify; plain HTTP on a + named/non-loopback host does not, and `WebMCPTesting` does not waive the rule. + A false result blocks entry into INVENTORY. +2. **Backend/CORS assumptions**: record every hardcoded/configured absolute backend + origin in `app.backendOrigins`, the exact development allow-list in + `app.corsAllowlist`, and choose an origin satisfying both. If the page never + finishes booting, capture console errors and failed requests before touching + tool code; `ERR_CONNECTION_REFUSED` or an exact-port CORS rejection is an + environment diagnosis. +3. **Coverage target + policy**: require `curated` or `parity`; never silently + default. Persist the normalized gate vocabulary, revision and fingerprint in + `pipeline.inventoryPolicy` before any area can be inventoried. +4. **Stack**: `package.json` deps (react/vue/@angular/next/astro/eleventy…) or the absence of one (static HTML). Record `app.stack` and `app.typescript`. -2. **Start command + base URL**: `dev`/`start` scripts, framework defaults - (`vite` → 5173, `next` → 3000, static → any file server). Verification needs a - working local run — if the app can't be started, append the blocker to - `pipeline.blockers` and surface it at the gate; don't silently proceed to a - verify phase that cannot run. -3. **Auth model**: none / session / role-based — plus **how a test session signs +5. **Start command + base URL**: record the actual command and observed URL. Ports + may be reassigned on shared hosts, so do not infer or pin a framework default. +6. **Auth model**: none / session / role-based — plus **how a test session signs in**, recorded per role under `app.authFixtures`: `obtain` (the exact steps — seed command, login route), `account`, and `env` (the env var **names** the fixture needs — never secret values in the manifest). The verify phase runs from this. Role-based apps need role-scoped registration (`integrate.md` §Auth) and a per-role verify pass. -4. **Git baseline**: `pipeline.baselineSha` = HEAD, `pipeline.baselineDirty` = +7. **Git baseline**: `pipeline.baselineSha` = HEAD, `pipeline.baselineDirty` = `git status --porcelain` paths. Dirty files are untouchable for the whole run. ## Building the area map @@ -54,17 +66,40 @@ dev/test-data-only verification; `"client"` may be batch-approved at the gate (`cleanup` recommended). `toolautosubmit` is banned for **both** mutation classes (ground rule 5). -**Skip** (do not inventory): login/logout/auth flows, payment execution, account -deletion, user management, anything irreversible, file uploads (v1), and pure -navigation agents can do anyway. - -## Tool budget, overlap, and priority (what keeps SaaS toolsets usable) +**Policy gates — use these exact classes.** Exclude auth/login/session/password/ +MFA/SSO; signup/registration/payment/billing/subscription; any tool that returns a +credential, token, key, JWT, signed URL or cookie; and irreversible deletion except +for a tool that opens the app's existing confirmation UI for the user. Everything +else is a product action and remains eligible, including creation of projects, +records, invitations, memberships and other ordinary domain objects. Pure navigation +an agent can already perform may still be omitted in `curated`, but it must be mapped +or reasoned about under `parity`. + +## Coverage target, tool budget, overlap, and priority + +`curated` selects high-value actions and may defer lower-value interaction classes, +but still emits an enumerated route→tool map with written omission reasons. +`parity` requires a route census: every interactive element on every authenticated +route maps to a tool or a written policy/technical reason. A count is never proof of +parity. Census deletes, drag/drop ordering, bulk/multi-select, table sorting, column +configuration, pagination, saved filters, invitations, memberships/permissions, +settings toggles, and canvas/viewer controls explicitly; these are the classes a +curated pass most often misses. + +No public source or measured client run establishes a universal safe number of tools +per page. Route-scoped registration reduces active tools, but the target client must +still enumerate and select them successfully; do not turn an arithmetic estimate into +a compatibility promise. Agents degrade when many similar tools compete. Enforce while drafting: -- **Budget**: aim for ≤15 tools active in any app state (app-wide + current view). +- **Curated budget**: aim for ≤15 tools active in any app state (app-wide + current view). If an area yields more candidates, keep the highest-value ones as `priority: 1` and mark the rest `priority: 2/3` — the gate decides which waves ship. +- **Parity capacity**: do not drop interactions to meet the curated budget. Partition + genuinely route-bound tools by route, record the active count, and verify the real + target client's enumeration/selection. A client-capacity failure is a named blocker, + not permission to claim 100% from a smaller count. - **Overlap rule**: no two tools whose descriptions could plausibly match the same user request. Merge them (one tool, richer schema) or sharpen both descriptions until they are disjoint. @@ -99,7 +134,7 @@ Agents degrade when many similar tools compete. Enforce while drafting: ## Writing manifest entries -Fill EVERY field of the v3 schema: +Fill EVERY field of the v4 schema: - `route` + `auth` (array of roles keying into `app.authFixtures`; verify runs once per role). @@ -116,5 +151,11 @@ Fill EVERY field of the v3 schema: The verify phase must be able to run from the manifest alone, without re-reading the codebase — that is what makes runs resumable by a different agent. -The completeness pass at the end of Phase 1: start the app (or read the rendered -nav), enumerate what a user can *do* per screen, and diff against the manifest. +For every area, persist the current `policyFingerprint` plus every policy exclusion +as `{ gate, reason }`; a zero-tool verdict without this provenance is invalid. If a +gate changes, apply SKILL.md's mechanical invalidation before resuming. + +Populate `routeCoverage` in both modes. The completeness pass starts the app, walks +each route under every recorded role, and diffs visible interactions against that +map. Under `parity` this is an element census; under `curated` it proves what was +selected and why the rest was deferred. diff --git a/skills/webmcpify/references/runtime.md b/skills/webmcpify/references/runtime.md index 8e97288..ff1e206 100644 --- a/skills/webmcpify/references/runtime.md +++ b/skills/webmcpify/references/runtime.md @@ -25,7 +25,7 @@ What it provides: |---|---| | `getModelContext()` | The ONLY place `document.modelContext` / deprecated `navigator.modelContext` is referenced — spec churn stays a one-file fix | | `isWebMCPAvailable()` | Feature detection — the app must work identically without WebMCP | -| `createToolScope(key, tools, options?)` | Registers a tool set under one AbortController; returns a **callable dispose handle** carrying `ready: Promise` (true = all registrations committed; false = no WebMCP / duplicate key / failure / disposed first — never rejects). Validates contracts BEFORE registering; **rolls back the whole scope** on any failure, including sync-throwing legacy `registerTool` (reported via `options.onError`, default `console.error` — NOT called when disposed before settling). An already-active key returns a no-op handle — safe under React StrictMode | +| `createToolScope(key, tools, options?)` | Registers a tool set under one AbortController; returns a **callable dispose handle** carrying `ready: Promise` (true = all registrations committed; false = no WebMCP / duplicate key / failure / disposed first — never rejects). Missing WebMCP remains a safe no-op, but development builds warn once with secure-origin/Chrome/flag diagnostics. Validates contracts BEFORE registering; wraps every imperative `execute()` so bare `null`/`undefined` becomes a structured error; **rolls back the whole scope** on any registration failure, including sync-throwing legacy `registerTool` (reported via `options.onError`, default `console.error` — NOT called when disposed before settling). An already-active key returns a no-op handle — safe under React StrictMode | | `dispatchAndWait(event, detail?, timeoutMs?)` | Bridges `execute()` to the app's own event/state flow. The dispatched detail carries `requestId` plus `signal` — an AbortSignal aborted on timeout; pass it to `fetch()` and skip state commits once aborted. Resolves only after the component confirms with an explicit **boolean** `ok`; a completion with missing/non-boolean `ok` **fails closed** to an `"ERROR: ..."` string, as do timeouts and `ok: false` (self-correction convention — never rejects). For tools whose confirmation involves a network round-trip (mailers, slow APIs), pass an explicit `timeoutMs` (e.g. `20_000`) instead of relying on the 10 s default | | `singleFlight(fn, busyMessage?)` | Serializes a tool's `execute`: while one call is in flight, further calls resolve immediately to a busy `"ERROR: ..."` string instead of racing shared UI state | @@ -34,6 +34,14 @@ Validation note: budget checks auto-enable when the bundler substitutes isn't `'production'`; unbundled projects default to off — pass `{ validate: true }` during development. +Result note: application tools must return a JSON-safe value. Bare `null` and +`undefined` are forbidden for imperative tools because current Chrome clients can +surface them as ambiguous success, the string `"null"`, or an execution-context +failure. The runtime converts an accidental absent result to +`{ ok: false, error: "…" }` (and logs it in development), but integrations should +return the intended structured result themselves. A browser-owned `null` from a +navigating declarative form is a different execution path and remains valid. + ## The completion contract (the part integrators get wrong) `dispatchAndWait` resolves when the component fires `tool-completion-` diff --git a/skills/webmcpify/references/security.md b/skills/webmcpify/references/security.md index 4bc473b..6ee429e 100644 --- a/skills/webmcpify/references/security.md +++ b/skills/webmcpify/references/security.md @@ -16,6 +16,13 @@ both directions. Design every tool as if it were a public, authenticated API end ## Checklist +**Scope vocabulary** +- [ ] Identity/tenancy/billing exclusions are written literally: no auth/login/ + session/password/MFA/SSO; no signup/registration/payment/billing/subscription; + no tool returning a credential, token, key, JWT, signed URL or cookie. + Creating or changing ordinary product objects is not "account creation" and + remains eligible. + **Trust boundary** - [ ] Every `execute()` calls only code paths the UI already uses — same endpoints, same validation, same authz, same rate limits. No new endpoints, no bypasses. @@ -25,11 +32,10 @@ both directions. Design every tool as if it were a public, authenticated API end **Human-in-the-loop** - [ ] No `toolautosubmit` on any state-changing form. -- [ ] No destructive/irreversible/payment tools at all in a first integration. - If the human explicitly insists later: an in-page manual confirmation the - **user** performs, PLUS a server-side two-step (short-lived confirm token). - No client-side API exists that can force an agent to confirm — never rely on - one. +- [ ] No payment/billing tool. No tool performs an irreversible destructive delete + directly; it may only open the app's existing confirmation UI for the **user** + to complete. Where the app uses a server-side two-step/short-lived confirm + token, preserve it. No client-side API can force an agent confirmation. - [ ] Initiation tools (`start_*_flow`) genuinely only navigate/open — they must not pre-execute any part of the mutation, and never carry `readOnlyHint`. diff --git a/skills/webmcpify/references/verify.md b/skills/webmcpify/references/verify.md index ad87a16..01262dd 100644 --- a/skills/webmcpify/references/verify.md +++ b/skills/webmcpify/references/verify.md @@ -2,14 +2,23 @@ ## Environment -- **Current Chrome** (the API moved during the trial — the old - `navigator.modelContextTesting` surface was removed 2026-07 in favor of - production `document.modelContext.getTools()/executeTool()`). +- **Current Chrome** (the API moved during the trial; verification probes + `document.modelContext`, never `navigator.modelContext`). A fresh container may + have no Chrome or virtual display: provision both explicitly and verify their + binaries before running. Do not assume `google-chrome` or `xvfb-run` exists. - Enable: `chrome://flags/#enable-webmcp-testing`, or launch with `--enable-features=WebMCP,WebMCPTesting` (covers both current and older builds). -- **Headed only** — WebMCP requires a visible tab by design. In CI, run under - `xvfb-run`. Headless will never work; don't heal toward it. -- App running locally via `app.startCommand`, against dev/test data only. +- **Headed only for this harness** — Chrome 150 exposed no `modelContext` in the + measured headless path. Run a real tab under a verified virtual display in CI; + don't heal toward headless. +- Set `WEBMCP_BASE_URL` to the observed verification origin and + `WEBMCP_PROFILE_DIR` to a dedicated writable user-data directory. Never pin a + checkout-specific `/work/...` path or assumed localhost port in the spec. Chrome + can ignore feature switches when another instance owns the same profile. +- App running via `app.startCommand`, against dev/test data only. If it never + boots, capture console errors and failed requests before touching tool code; + absolute backend URLs and exact-port CORS allow-lists are common environment + failures. - Each tool's manifest entry tells you where and how: `route` (navigate there), `auth` (sign in with the recorded test fixture; verify under EACH role for role-scoped tools), `examples` (what to execute), `expect` (what to assert), @@ -17,13 +26,12 @@ ## The enumeration/execution surface (probe, don't assume) -In the page context, prefer the production surface and fall back for older builds: +In the page context, probe the current production surface: ```js -const mc = document.modelContext ?? navigator.modelContext; -const tools = mc?.getTools - ? await mc.getTools() - : await navigator.modelContextTesting?.listTools(); // removed 2026-07; legacy only +const mc = document.modelContext; +if (!window.isSecureContext || !mc) throw new Error('insecure origin or unsupported test environment'); +const tools = await mc.getTools(); ``` Contract facts that generated assertions MUST respect: @@ -55,13 +63,11 @@ as the expected property in the actual target Chrome build. ## Per-tool checks 1. Registered (poll — registration is async) with the expected name, the (parsed) - schema, **and** the manifest `annotations` on the enumerated tool. The legacy - `modelContextTesting` fallback cannot enumerate annotations — skip that - assertion there and note the gap in the report. + schema, **and** the manifest `annotations` on the enumerated tool. 2. Valid example executes: assert the result per `expect` — `expect.result` as a - substring of the resolved string, or `expect.navigation` as the destination - when `executeTool` resolves `null` (it navigated) — **and** the `expect.ui` - state as a **delta** (capture the relevant state *before* executing; mere + substring of the resolved string, plus `expect.navigation` when a declarative + submit navigates or an imperative tool defers the app's existing route action — + **and** the `expect.ui` state as a **delta** (capture the relevant state *before* executing; mere visibility of something already on screen proves nothing). A tool that reports success without the UI changing is a **fail** (UI-settled rule). Because executions can navigate, restore the manifest `route` in `beforeEach`, not @@ -99,7 +105,9 @@ export default defineConfig({ workers: 1, // one shared headed Chrome — never parallelize }); EOF -WEBMCP_SPEC_DIR=/.webmcpify WEBMCP_BASE_URL=http://localhost:5173 \ +WEBMCP_SPEC_DIR=/.webmcpify \ +WEBMCP_BASE_URL= \ +WEBMCP_PROFILE_DIR= \ NODE_PATH=/tmp/webmcpify-harness/node_modules npx playwright test ``` @@ -109,6 +117,11 @@ tooling ignores `NODE_PATH`, symlink instead: (and make sure it isn't committed). Note in the report that verification ran from a standalone harness. +For ChatGPT's separate built-in-browser experience—named **Site tools**, with +account/model availability and page-lifetime behavior—use `references/client.md`. +Chrome harness success alone does not prove a specific ChatGPT account can use the +tools. + **Alternative:** Puppeteer ships a first-class experimental WebMCP API (https://pptr.dev/guides/webmcp) — prefer it when the target repo already uses Puppeteer. diff --git a/skills/webmcpify/templates/webmcp.d.ts b/skills/webmcpify/templates/webmcp.d.ts index 1f43df1..e08d5d1 100644 --- a/skills/webmcpify/templates/webmcp.d.ts +++ b/skills/webmcpify/templates/webmcp.d.ts @@ -40,6 +40,13 @@ interface ModelContextToolAnnotations { untrustedContentHint?: boolean; } +type ModelContextToolResult = + | string + | number + | boolean + | Record + | unknown[]; + interface ModelContext extends EventTarget { registerTool( tool: ModelContextTool, @@ -77,7 +84,8 @@ interface ModelContextTool { * a promise, so a sync implementation still fulfills this type at runtime; * declare it async for type fidelity. */ - execute(input: Record): Promise; + /** JSON-safe result; bare null/undefined are intentionally excluded. */ + execute(input: Record): Promise; annotations?: ModelContextToolAnnotations; } diff --git a/skills/webmcpify/templates/webmcp.spec.ts b/skills/webmcpify/templates/webmcp.spec.ts index e4ae02a..1d1649e 100644 --- a/skills/webmcpify/templates/webmcp.spec.ts +++ b/skills/webmcpify/templates/webmcp.spec.ts @@ -30,22 +30,29 @@ * below show the complete patterns with REAL assertions — generated blocks must * assert, never comment out. * - * Requirements: real Chrome, HEADED (WebMCP needs a visible tab — headless will - * never work; use xvfb-run in CI). Enumeration/execution uses the production - * document.modelContext.getTools()/executeTool() surface (Chrome 2026-07+), with a - * probe fallback to the removed navigator.modelContextTesting for older builds. + * Requirements: real current Chrome, HEADED (headless exposes no modelContext in + * the supported verification path), a virtual display when needed, and a dedicated + * user-data directory. Enumeration/execution uses the production + * document.modelContext.getTools()/executeTool() surface (Chrome 2026-07+). * Alternative harness: Puppeteer's first-class WebMCP API (pptr.dev/guides/webmcp). */ import { chromium, expect, test } from '@playwright/test'; import type { BrowserContext, Page } from '@playwright/test'; -const BASE_URL = process.env.WEBMCP_BASE_URL ?? 'http://localhost:5173'; +function requiredEnv(name: 'WEBMCP_BASE_URL' | 'WEBMCP_PROFILE_DIR'): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required; do not pin a host port or profile path in the spec`); + return value; +} + +const BASE_URL = requiredEnv('WEBMCP_BASE_URL'); +const PROFILE_DIR = requiredEnv('WEBMCP_PROFILE_DIR'); let context: BrowserContext; let page: Page; test.beforeAll(async () => { - context = await chromium.launchPersistentContext('', { + context = await chromium.launchPersistentContext(PROFILE_DIR, { channel: 'chrome', headless: false, args: ['--enable-features=WebMCP,WebMCPTesting'], @@ -66,11 +73,9 @@ async function listTools(p: Page): Promise< }> > { return p.evaluate(async () => { - const mc = (document as any).modelContext ?? (navigator as any).modelContext; + const mc = (document as any).modelContext; if (mc?.getTools) return mc.getTools(); - const legacy = (navigator as any).modelContextTesting; // removed 2026-07; older builds only - if (legacy?.listTools) return legacy.listTools(); - throw new Error('No WebMCP enumeration surface — wrong Chrome build or flags'); + throw new Error('No document.modelContext enumeration surface — insecure origin, headless/wrong Chrome, reused profile, or missing flag'); }); } @@ -82,16 +87,14 @@ async function listTools(p: Page): Promise< async function executeTool(p: Page, name: string, args: object): Promise { return p.evaluate( async ({ name, args }) => { - const mc = (document as any).modelContext ?? (navigator as any).modelContext; + const mc = (document as any).modelContext; if (mc?.getTools && mc?.executeTool) { const tools = await mc.getTools(); const tool = tools.find((t: { name: string }) => t.name === name); if (!tool) throw new Error(`tool ${name} is not registered`); return mc.executeTool(tool, JSON.stringify(args)); } - const legacy = (navigator as any).modelContextTesting; - if (legacy?.executeTool) return legacy.executeTool(name, JSON.stringify(args)); - throw new Error('No WebMCP execution surface — wrong Chrome build or flags'); + throw new Error('No document.modelContext execution surface — insecure origin, headless/wrong Chrome, reused profile, or missing flag'); }, { name, args }, ); @@ -111,22 +114,17 @@ async function waitForTool(p: Page, name: string, timeoutMs = 5000): Promise { - return p.evaluate(() => { - const mc = (document as any).modelContext ?? (navigator as any).modelContext; - return !!mc?.getTools; - }); -} - -test('WebMCP is available in the test environment', async () => { +test('verification origin is secure and WebMCP is available', async () => { await page.goto(BASE_URL); - const available = await page.evaluate( - () => !!(document as any).modelContext || !!(navigator as any).modelContext, - ); - expect(available, 'Enable chrome://flags/#enable-webmcp-testing and use current Chrome').toBe( - true, - ); + const probe = await page.evaluate(() => ({ + secureContext: window.isSecureContext, + hasDocumentModelContext: !!(document as any).modelContext, + })); + expect(probe.secureContext, `Verification origin must be secure: ${BASE_URL}`).toBe(true); + expect( + probe.hasDocumentModelContext, + 'Use current headed Chrome, a dedicated profile, and enable chrome://flags/#enable-webmcp-testing', + ).toBe(true); }); // ── Generated per manifest tool ────────────────────────────────────────────── @@ -147,18 +145,9 @@ test.describe('search_tickets', () => { const tool = tools.find((t) => t.name === 'search_tickets')!; const schema = JSON.parse(tool.inputSchema ?? '{}'); // stringified → parse first expect(schema.required).toContain('query'); // manifest: inputSchema - if (await hasModernSurface(page)) { - // manifest: annotations — assert exactly what the manifest recorded - expect(tool.annotations?.readOnlyHint).toBe(true); - expect(tool.annotations?.untrustedContentHint).toBe(true); - } else { - // Legacy modelContextTesting fallback cannot enumerate annotations — - // skip the assertion and record the gap in the report. - test.info().annotations.push({ - type: 'webmcpify', - description: 'annotations not enumerable on this Chrome build — assertion skipped', - }); - } + // manifest: annotations — assert exactly what the manifest recorded + expect(tool.annotations?.readOnlyHint).toBe(true); + expect(tool.annotations?.untrustedContentHint).toBe(true); }); test('executes the valid example and changes the UI', async () => { @@ -242,7 +231,7 @@ test.describe('get_page_summary', () => { } catch (err) { // Rejected: acceptable only as a validation rejection — a missing surface // or unregistered tool is a real failure, not a pass. - expect(String(err)).not.toMatch(/No WebMCP|is not registered/); + expect(String(err)).not.toMatch(/No document\.modelContext|is not registered/); } }); }); diff --git a/skills/webmcpify/templates/webmcpify.js b/skills/webmcpify/templates/webmcpify.js index d069c2d..812cae0 100644 --- a/skills/webmcpify/templates/webmcpify.js +++ b/skills/webmcpify/templates/webmcpify.js @@ -53,6 +53,7 @@ export function isWebMCPAvailable() { } const scopes = new Map(); +let warnedMissingModelContext = false; /** * @param {() => void} dispose @@ -87,10 +88,20 @@ function makeHandle(dispose, ready) { */ export function createToolScope(key, tools, options) { const mc = getModelContext(); - if (!mc) return makeHandle(() => {}, Promise.resolve(false)); + const validate = shouldValidate(options); + if (!mc) { + if (validate && !warnedMissingModelContext) { + warnedMissingModelContext = true; + console.warn( + `webmcpify: WebMCP is unavailable for scope "${key}". Check window.isSecureContext, the verification origin, current Chrome, and the WebMCP feature flag. Registration remains a safe no-op.`, + ); + } + return makeHandle(() => {}, Promise.resolve(false)); + } if (scopes.has(key)) return makeHandle(() => {}, Promise.resolve(false)); - if (shouldValidate(options)) for (const tool of tools) validateTool(tool); + if (validate) for (const tool of tools) validateTool(tool); + const guardedTools = tools.map((tool) => guardToolResult(tool, validate)); const controller = new AbortController(); scopes.set(key, controller); @@ -110,7 +121,9 @@ export function createToolScope(key, tools, options) { try { // Legacy registerTool implementations throw synchronously instead of // rejecting — normalize so the rollback path below covers both. - registrations = Promise.all(tools.map((tool) => mc.registerTool(tool, registerOptions))); + registrations = Promise.all( + guardedTools.map((tool) => mc.registerTool(tool, registerOptions)), + ); } catch (error) { registrations = Promise.reject(error); } @@ -135,6 +148,27 @@ export function createToolScope(key, tools, options) { }, ready); } +/** + * Imperative tools must never hand Chrome an ambiguous absent result. + * @param {object & { name: string, execute: (input: Record) => unknown }} tool + * @param {boolean} reportInDevelopment + * @returns {object} + */ +function guardToolResult(tool, reportInDevelopment) { + return { + ...tool, + async execute(input) { + const result = await tool.execute(input); + if (result !== null && result !== undefined) return result; + const error = + `Tool "${tool.name}" returned ${result === null ? 'null' : 'undefined'}. ` + + 'Return a JSON-safe structured result before any route-changing UI action.'; + if (reportInDevelopment) console.error(`webmcpify: ${error}`); + return { ok: false, error }; + }, + }; +} + /** * Bridge execute() to the app's own event/state flow. The dispatched detail * carries `{ ...detail, requestId, signal }` — `signal` is an AbortSignal aborted diff --git a/skills/webmcpify/templates/webmcpify.ts b/skills/webmcpify/templates/webmcpify.ts index 49f710f..f7f91ac 100644 --- a/skills/webmcpify/templates/webmcpify.ts +++ b/skills/webmcpify/templates/webmcpify.ts @@ -82,6 +82,7 @@ export function isWebMCPAvailable(): boolean { } const scopes = new Map(); +let warnedMissingModelContext = false; function makeHandle(dispose: () => void, ready: Promise): ToolScopeHandle { const handle = dispose as ToolScopeHandle; @@ -111,10 +112,20 @@ export function createToolScope( options?: ToolScopeOptions, ): ToolScopeHandle { const mc = getModelContext(); - if (!mc) return makeHandle(() => {}, Promise.resolve(false)); + const validate = shouldValidate(options); + if (!mc) { + if (validate && !warnedMissingModelContext) { + warnedMissingModelContext = true; + console.warn( + `webmcpify: WebMCP is unavailable for scope "${key}". Check window.isSecureContext, the verification origin, current Chrome, and the WebMCP feature flag. Registration remains a safe no-op.`, + ); + } + return makeHandle(() => {}, Promise.resolve(false)); + } if (scopes.has(key)) return makeHandle(() => {}, Promise.resolve(false)); - if (shouldValidate(options)) for (const tool of tools) validateTool(tool); + if (validate) for (const tool of tools) validateTool(tool); + const guardedTools = tools.map((tool) => guardToolResult(tool, validate)); const controller = new AbortController(); scopes.set(key, controller); @@ -136,7 +147,9 @@ export function createToolScope( try { // Legacy registerTool implementations throw synchronously instead of // rejecting — normalize so the rollback path below covers both. - registrations = Promise.all(tools.map((tool) => mc.registerTool(tool, registerOptions))); + registrations = Promise.all( + guardedTools.map((tool) => mc.registerTool(tool, registerOptions)), + ); } catch (error) { registrations = Promise.reject(error); } @@ -161,6 +174,22 @@ export function createToolScope( }, ready); } +/** Imperative tools must never hand Chrome an ambiguous absent result. */ +function guardToolResult(tool: ModelContextTool, reportInDevelopment: boolean): ModelContextTool { + return { + ...tool, + async execute(input: Record): Promise { + const result = await tool.execute(input); + if (result !== null && result !== undefined) return result; + const error = + `Tool "${tool.name}" returned ${result === null ? 'null' : 'undefined'}. ` + + 'Return a JSON-safe structured result before any route-changing UI action.'; + if (reportInDevelopment) console.error(`webmcpify: ${error}`); + return { ok: false, error }; + }, + }; +} + /** * Bridge execute() to the app's own event/state flow. The dispatched detail * carries `{ ...detail, requestId, signal }` — `signal` is an AbortSignal aborted diff --git a/tests/parity.test.mjs b/tests/parity.test.mjs index 73c49f7..fcc4be7 100644 --- a/tests/parity.test.mjs +++ b/tests/parity.test.mjs @@ -174,14 +174,54 @@ test('createToolScope: happy path — ready resolves true; duplicate key resolve test('createToolScope: no WebMCP — no-op handle, ready false', async () => { const results = []; - for (const [, mod] of variants) { - setModelContext(undefined); - const handle = mod.createToolScope('no-mc', []); - results.push({ ready: await handle.ready, callable: typeof handle === 'function' }); - handle(); + const warnings = []; + const originalWarn = console.warn; + console.warn = (message) => warnings.push(String(message)); + try { + for (const [tag, mod] of variants) { + setModelContext(undefined); + const handle = mod.createToolScope(`no-mc-${tag}`, [], { validate: true }); + results.push({ ready: await handle.ready, callable: typeof handle === 'function' }); + handle(); + } + } finally { + console.warn = originalWarn; } assert.deepEqual(results[0], results[1]); assert.deepEqual(results[0], { ready: false, callable: true }); + assert.equal(warnings.length, 2, 'each runtime variant warns once in development'); + for (const warning of warnings) assert.match(warning, /window\.isSecureContext/); +}); + +test('createToolScope: guards bare null and undefined tool results with structured errors', async () => { + const results = []; + for (const [tag, mod] of variants) { + freshWindow(); + const registered = []; + setModelContext({ registerTool: async (tool) => registered.push(tool) }); + const handle = mod.createToolScope( + `result-guard-${tag}`, + [ + { ...validTool('null_tool'), execute: async () => null }, + { ...validTool('undefined_tool'), execute: async () => undefined }, + { ...validTool('object_tool'), execute: async () => ({ opened: true }) }, + ], + { validate: false }, + ); + assert.equal(await handle.ready, true); + results.push({ + nullResult: await registered[0].execute({}), + undefinedResult: await registered[1].execute({}), + objectResult: await registered[2].execute({}), + }); + handle(); + } + assert.deepEqual(results[0], results[1]); + assert.deepEqual(results[0].objectResult, { opened: true }); + assert.equal(results[0].nullResult.ok, false); + assert.match(results[0].nullResult.error, /returned null/); + assert.equal(results[0].undefinedResult.ok, false); + assert.match(results[0].undefinedResult.error, /returned undefined/); }); test('createToolScope: sync-throwing registerTool — one onError, ready false, key reusable', async () => { diff --git a/tests/versions.test.mjs b/tests/versions.test.mjs index a0af846..e31de7e 100644 --- a/tests/versions.test.mjs +++ b/tests/versions.test.mjs @@ -20,6 +20,15 @@ const versions = [ ['skill.json', read('skill.json').version], ]; +const descriptions = [ + ['package.json', read('package.json').description], + ['.claude-plugin/plugin.json', read('.claude-plugin/plugin.json').description], + ['.claude-plugin/marketplace.json', read('.claude-plugin/marketplace.json').plugins?.[0]?.description], + ['.cursor-plugin/plugin.json', read('.cursor-plugin/plugin.json').description], + ['gemini-extension.json', read('gemini-extension.json').description], + ['skill.json', read('skill.json').description], +]; + test('every distribution manifest defines a version', () => { for (const [file, version] of versions) { assert.equal(typeof version, 'string', `${file} must define a version string`); @@ -33,3 +42,13 @@ test('all distribution manifests carry the same version as package.json', () => assert.equal(version, reference, `${file} version must equal package.json's ${reference}`); } }); + +test('public distribution descriptions carry the same coverage contract', () => { + const [, reference] = descriptions[0]; + for (const [file, description] of descriptions) { + assert.equal(description, reference, `${file} description must equal package.json`); + } + assert.match(reference, /WebMCP agent skill/); + assert.match(reference, /curated core coverage/); + assert.match(reference, /route-by-route parity/); +});