From 481747a07ad841f04a473b9329eacd6c3da406be Mon Sep 17 00:00:00 2001 From: 0xpantera <0xpantera@proton.me> Date: Wed, 24 Jun 2026 17:16:13 +0200 Subject: [PATCH] test: add WalletBeat safety matrix gate --- docs/WALLETBEAT-SAFETY-MATRIX.md | 77 +++++++++++++ docs/walletbeat-qa.md | 14 ++- .../issue-95-walletbeat-safety-matrix.md | 80 ++++++++++++++ package.json | 3 +- scripts/walletbeat-safety-matrix-qa.mjs | 103 ++++++++++++++++++ 5 files changed, 273 insertions(+), 4 deletions(-) create mode 100644 docs/WALLETBEAT-SAFETY-MATRIX.md create mode 100644 execplans/issue-95-walletbeat-safety-matrix.md create mode 100644 scripts/walletbeat-safety-matrix-qa.mjs diff --git a/docs/WALLETBEAT-SAFETY-MATRIX.md b/docs/WALLETBEAT-SAFETY-MATRIX.md new file mode 100644 index 0000000..1c68807 --- /dev/null +++ b/docs/WALLETBEAT-SAFETY-MATRIX.md @@ -0,0 +1,77 @@ +# WalletBeat safety matrix + +This document is the tracking gate for WalletBeat scam-alert and transaction-simulation coverage. +It exists to prevent Deckard from treating safe refusal as full scam/simulation support. + +Status values: + +- **supported** — Deckard has positive behavior for the fixture. +- **safe refusal** — Deckard intentionally refuses the request and does not sign/broadcast it. +- **tracked gap** — WalletBeat expects positive safety behavior that Deckard does not yet implement; a linked issue owns the gap. +- **blocked** — depends on a larger architectural prerequisite. + +Closure rule for #95: every fixture in the pinned WalletBeat checkout must appear below and every +`tracked gap` / `blocked` row must link to a GitHub issue. + +## Current feature boundary + +Deckard currently provides: + +- reviewed `personal_sign` and `eth_signTypedData_v4` +- explicit raw `eth_sign` refusal +- reviewed native ETH send +- reviewed ERC-20 `transfer(address,uint256)` and `approve(address,uint256)` classification +- non-atomic EIP-5792 batches made only from current clear-signable transaction shapes +- fail-closed refusal for unknown/arbitrary calldata + +Deckard does **not** yet provide: + +- address reputation or known-scam lists +- recent-contract or previous-interaction context +- first-class infinite-approval / Permit risk warnings +- revm-style transaction simulation +- token/NFT asset-delta rendering +- misleading-selector / fake-airdrop / volatile-outcome / guaranteed-revert detection + +## Scam-alert fixtures + +Source: `.walletbeat/walletbeat/src/constants/test-scam-alerts.ts`. + +| WalletBeat fixture | Risk type | Deckard status | Current behavior | Required follow-up | +| --- | --- | --- | --- | --- | +| `recent-contract-1` | recent deploy | tracked gap | Deckard can refuse/route unknown calldata, but does not know whether a contract was recently deployed. | #150, #73 | +| `previous-interaction-1` | previous interaction | tracked gap | Deckard does not track or display per-wallet contract interaction history. | #150 | +| `wallet-own-1` | known scam / custom recipient | tracked gap | Deckard can review a native send, but does not score arbitrary recipients as suspicious. | #149, #135 | +| `known-scam-eth-send` | known scam address | tracked gap | Deckard does not maintain a scam-address fixture/list or reputation source. | #149, #135 | +| `allow-infinite-usdc` | infinite ERC-20 approval | tracked gap | Deckard classifies ERC-20 `approve`, but does not yet elevate unlimited allowance/spender risk as a dedicated warning. | #151, #135 | +| `allow-infinite-permit` | infinite permit signature | tracked gap | Deckard supports EIP-712 review and chain-id mismatch denial, but does not yet render Permit-specific owner/spender/value/deadline risk rows. | #151, #135 | + +## Transaction-simulation fixtures + +Source: `.walletbeat/walletbeat/src/components/Tabs/TransactionSimulationsTab.svelte`. + +| WalletBeat fixture | Function | Deckard status | Current behavior | Required follow-up | +| --- | --- | --- | --- | --- | +| `erc20-mint` | `mintHundred()` | tracked gap | Arbitrary mint calldata is refused today; Deckard does not simulate token mint deltas. | #152, #73, #74 | +| `erc721-mint` | `mintOne()` | tracked gap | ERC-721 mint calldata is refused; Deckard does not simulate NFT mint deltas. | #152, #73, #74 | +| `erc1155-mint` | `mintOne()` | tracked gap | ERC-1155 mint calldata is refused; Deckard does not simulate semi-fungible token deltas. | #152, #73, #74 | +| `erc20-transfer` | `transfer(address,uint256)` | tracked gap | Deckard can classify ERC-20 transfer calldata, but simulation requires pre/post asset delta rendering instead of selector-only display. | #152, #73, #74 | +| `erc721-transfer` | `safeTransferFrom(address,address,uint256)` | tracked gap | ERC-721 transfer calldata is refused; Deckard does not render NFT ownership deltas. | #152, #73, #74 | +| `erc1155-transfer` | `safeTransferFrom(address,address,uint256,uint256,bytes)` | tracked gap | ERC-1155 transfer calldata is refused; Deckard does not render balance deltas. | #152, #73, #74 | +| `all-token-transfer` | `simulateFunctionV1()` | tracked gap | Mixed token effects require real simulation and multi-asset rendering. | #152, #153, #73, #74 | +| `misleading-selector` | selector looks like `transfer(address,uint256)` | tracked gap | This is a known dangerous ambiguity: selector-only classification cannot prove actual contract semantics. Deckard needs simulation/metadata before claiming support. | #153, #73, #74 | +| `fake-airdrop` | `claim()` drains while emitting misleading mint-like event | tracked gap | Requires state-diff simulation and event/balance reconciliation. | #153, #73 | +| `volatile-outcome` | block-dependent mint/burn | tracked gap | Requires simulation volatility/race warning and possibly repeated preflight near signing time. | #153, #73 | +| `failing-transaction` | guaranteed revert | tracked gap | Requires preflight revert detection before signing. | #153, #73 | + +## QA gate expectations + +`pnpm run qa:walletbeat:safety` validates that: + +1. every WalletBeat scam-alert fixture id appears in this matrix; +2. every WalletBeat transaction-simulation fixture id appears in this matrix; +3. every `tracked gap` or `blocked` row has at least one linked issue; +4. the matrix has no `untracked gap` marker; +5. the current local-chain QA lane continues to prove the already-supported/refused primitives. + +This matrix should be updated before bumping `DECKARD_WALLETBEAT_REF` if WalletBeat adds or renames fixtures. diff --git a/docs/walletbeat-qa.md b/docs/walletbeat-qa.md index 5148951..cbf69f6 100644 --- a/docs/walletbeat-qa.md +++ b/docs/walletbeat-qa.md @@ -25,6 +25,12 @@ Run headed while debugging: npm run qa:walletbeat:headed ``` +Validate the scam-alert / transaction-simulation safety matrix: + +```sh +npm run qa:walletbeat:safety +``` + The script: - clones a pinned WalletBeat beta checkout into `.walletbeat/` @@ -60,9 +66,11 @@ Useful overrides: ## Scope -This lane is not a full WalletBeat pass. It intentionally excludes transaction, signature, -batch-call, scam-alert, and simulation tabs until those flows use reviewed Deckard approval UI and -local-chain-only fixtures. +The default safe provider lane is not a full WalletBeat pass. Transaction, signature, and batch-call +coverage live in dedicated lanes. Scam-alert and transaction-simulation coverage is tracked by +`docs/WALLETBEAT-SAFETY-MATRIX.md`; that matrix is deliberately not a claim of full support. It +forces every WalletBeat safety fixture to be classified as supported, safely refused, or linked to a +follow-up issue before #95 can close. Tracked follow-up work: diff --git a/execplans/issue-95-walletbeat-safety-matrix.md b/execplans/issue-95-walletbeat-safety-matrix.md new file mode 100644 index 0000000..af20eba --- /dev/null +++ b/execplans/issue-95-walletbeat-safety-matrix.md @@ -0,0 +1,80 @@ +# Issue #95 — WalletBeat scam/simulation safety matrix + +## 1. Title + +Add a WalletBeat scam-alert / transaction-simulation safety matrix and QA gate. + +## 2. Context + +WalletBeat's scam-alert and transaction-simulation tabs are not simple RPC compatibility checks. They probe higher-level wallet safety behavior: + +- reputation / suspicious-recipient warnings +- contract age and previous interaction context +- infinite approval and permit-signature risk display +- token/NFT asset-delta simulation +- adversarial outcome detection such as misleading selectors, fake airdrops, volatile outcomes, and guaranteed reverts + +Deckard now supports a reviewed browser bridge for accounts, message signing, native sends, ERC-20 `transfer`/`approve`, EIP-5792 non-atomic batches, and a signerd-backed local-chain WalletBeat QA lane. It does **not** yet provide a full scam reputation engine or revm-style asset-delta simulation. + +## 3. Decision + +Do not close #95 by pretending unsupported simulation/scam behavior is implemented. Close it only as a coverage/gap-control gate: + +1. Every WalletBeat scam/simulation fixture is listed in a committed matrix. +2. Every unsupported positive behavior has a linked follow-up issue. +3. A QA script fails if WalletBeat adds a new fixture without matrix coverage. +4. Current Deckard behavior is described honestly: supported, safe refusal, tracked gap, or blocked on architecture. + +## 4. Acceptance criteria + +- [ ] Read all WalletBeat scam-alert fixtures. +- [ ] Read all WalletBeat transaction-simulation fixtures. +- [ ] Build a fixture-by-fixture Deckard status matrix. +- [ ] Add a local QA script that validates matrix completeness against the pinned WalletBeat checkout. +- [ ] Assert unsafe/unsupported cases are refused, not signed, where current automation can do so safely. +- [ ] Create/link GitHub issues for every unsupported positive feature. +- [ ] Update GitHub Project so those follow-ups are visible. +- [ ] Do not close #95 until the matrix has no `untracked gap` rows. + +## 5. Follow-up issue map + +- #149 — address reputation and suspicious-recipient warnings. +- #150 — contract age and previous-interaction context. +- #151 — infinite approval and permit risk warnings. +- #152 — token and NFT asset-delta rendering. +- #153 — adversarial simulation outcome warnings. +- Existing #73 — revm preflight/simulation engine foundation. +- Existing #74 — verified token metadata foundation. +- Existing #135 — rules engine v2 / policy integration. + +## 6. Verification plan + +Focused: + +- `pnpm run qa:walletbeat:safety` +- `pnpm run qa:walletbeat:local-chain` + +Full DoD before PR: + +- `cargo fmt --all --check` +- `just check` +- `cargo test --workspace` +- `pnpm run qa:extension` +- `pnpm run qa:extension:real` +- `pnpm run qa:walletbeat` +- `pnpm run qa:walletbeat:signatures` +- `pnpm run qa:walletbeat:transactions` +- `pnpm run qa:walletbeat:local-chain` +- `pnpm run qa:walletbeat:safety` +- `git diff --check` + +## 7. Status + +- [x] Branch created from merged `origin/main`. +- [x] Issue #95 and WalletBeat fixture files read. +- [x] Follow-up issues created. +- [x] Safety matrix committed. +- [x] QA gate committed. +- [x] GitHub Project updated. +- [x] Full local DoD. +- [ ] PR opened and CI checked. diff --git a/package.json b/package.json index 7e5709c..fc05fdc 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "qa:walletbeat:transactions": "node scripts/walletbeat-transactions-qa.mjs", "qa:walletbeat:transactions:headed": "DECKARD_WALLETBEAT_HEADED=1 node scripts/walletbeat-transactions-qa.mjs", "qa:walletbeat:local-chain": "node scripts/walletbeat-local-chain-qa.mjs", - "qa:walletbeat:local-chain:headed": "DECKARD_WALLETBEAT_HEADED=1 node scripts/walletbeat-local-chain-qa.mjs" + "qa:walletbeat:local-chain:headed": "DECKARD_WALLETBEAT_HEADED=1 node scripts/walletbeat-local-chain-qa.mjs", + "qa:walletbeat:safety": "node scripts/walletbeat-safety-matrix-qa.mjs" }, "devDependencies": { "@playwright/test": "^1.61.0" diff --git a/scripts/walletbeat-safety-matrix-qa.mjs b/scripts/walletbeat-safety-matrix-qa.mjs new file mode 100644 index 0000000..ec41b34 --- /dev/null +++ b/scripts/walletbeat-safety-matrix-qa.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const repoRoot = path.resolve(__dirname, '..'); +const walletbeatRoot = path.join(repoRoot, '.walletbeat', 'walletbeat'); +const matrixPath = path.join(repoRoot, 'docs', 'WALLETBEAT-SAFETY-MATRIX.md'); +const scamPath = path.join(walletbeatRoot, 'src', 'constants', 'test-scam-alerts.ts'); +const simulationPath = path.join(walletbeatRoot, 'src', 'components', 'Tabs', 'TransactionSimulationsTab.svelte'); + +const requiredIssues = new Set(['#73', '#74', '#135', '#149', '#150', '#151', '#152', '#153']); + +function read(file) { + return fs.readFileSync(file, 'utf8'); +} + +function ensureWalletbeatCheckout() { + for (const file of [scamPath, simulationPath]) { + if (!fs.existsSync(file)) { + throw new Error(`WalletBeat checkout missing ${path.relative(repoRoot, file)}. Run pnpm run qa:walletbeat first to populate .walletbeat/.`); + } + } +} + +function scamFixtureIds(source) { + return [...source.matchAll(/id:\s*'([^']+)'/g)].map((match) => match[1]); +} + +function simulationFixtureIds(source) { + const typeMatch = source.match(/export type TransactionSimulationSubTab =([\s\S]*?);/); + if (!typeMatch) throw new Error('Could not find TransactionSimulationSubTab union'); + return [...typeMatch[1].matchAll(/'([^']+)'/g)].map((match) => match[1]); +} + +function matrixRows(markdown) { + const rows = new Map(); + for (const line of markdown.split('\n')) { + if (!line.startsWith('| `')) continue; + const cells = line.split('|').slice(1, -1).map((cell) => cell.trim()); + if (cells.length < 5) continue; + const id = cells[0].replace(/^`|`$/g, ''); + rows.set(id, { status: cells[2], followup: cells[4], line }); + } + return rows; +} + +function issuesIn(text) { + return new Set([...text.matchAll(/#\d+/g)].map((match) => match[0])); +} + +function main() { + ensureWalletbeatCheckout(); + const matrix = read(matrixPath); + const rows = matrixRows(matrix); + const expected = [ + ...scamFixtureIds(read(scamPath)), + ...simulationFixtureIds(read(simulationPath)), + ]; + const failures = []; + + for (const id of expected) { + if (!rows.has(id)) failures.push(`missing matrix row for WalletBeat fixture ${id}`); + } + for (const id of rows.keys()) { + if (!expected.includes(id)) failures.push(`matrix row ${id} is not present in pinned WalletBeat fixtures`); + } + for (const [id, row] of rows) { + if (/untracked gap/i.test(row.status) || /untracked gap/i.test(row.followup)) { + failures.push(`matrix row ${id} still contains an untracked gap marker`); + } + } + + for (const [id, row] of rows) { + if (/tracked gap|blocked/i.test(row.status) && !/#\d+/.test(row.followup)) { + failures.push(`matrix row ${id} is ${row.status} but has no linked follow-up issue`); + } + } + + const matrixIssues = issuesIn(matrix); + for (const issue of requiredIssues) { + if (!matrixIssues.has(issue)) failures.push(`matrix is missing required issue link ${issue}`); + } + + if (failures.length > 0) { + console.error('WalletBeat safety matrix QA failed:'); + for (const failure of failures) console.error(`- ${failure}`); + process.exit(1); + } + + const tracked = [...rows.values()].filter((row) => /tracked gap/i.test(row.status)).length; + const safe = [...rows.values()].filter((row) => /safe refusal/i.test(row.status)).length; + const supported = [...rows.values()].filter((row) => /supported/i.test(row.status)).length; + console.log('WalletBeat safety matrix QA passed:'); + console.log(`fixtures covered: ${expected.length}`); + console.log(`supported: ${supported}`); + console.log(`safe refusal: ${safe}`); + console.log(`tracked gaps: ${tracked}`); +} + +main();