From 2c7aad1c0c6bdbf4202b3e27044a9dd695882f8d Mon Sep 17 00:00:00 2001 From: elhoim Date: Sun, 26 Jul 2026 17:19:14 +0000 Subject: [PATCH] feat(interceptor): add measured zoom for appearance verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Images are downscaled to ~1568px on the long edge before the model sees them. On a 2000px-wide page a 40px logo arrives as a smudge, and cropping the delivered screenshot cannot recover it — the detail was destroyed upstream. That is the failure mode behind the appearance-vs-existence rule: a screenshot proves an element exists, not what it looks like. Zoom.ts takes absolute coordinates in the ORIGINAL image, crops from the full-resolution source, and magnifies that region to the image budget, so the pixels spent land on the thing under test. Nearest-neighbour on the upscale, so it magnifies without inventing detail. It never downscales: a region already past the budget is passed through at native size. Prefers `magick` (the binary Capture.sh and VerifyImageProbe.ts already use) and falls back to `ffmpeg` so it works on hosts without ImageMagick. Out-of-bounds, zero-area and negative-origin crops exit 2 with the source dimensions in the message rather than emitting a misleading image. (cherry picked from commit 6bbe5e242686daa82ce2a82e109a3959de6d69ce) --- LifeOS/install/skills/Interceptor/SKILL.md | 3 +- .../install/skills/Interceptor/Tools/Zoom.ts | 148 ++++++++++++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100755 LifeOS/install/skills/Interceptor/Tools/Zoom.ts diff --git a/LifeOS/install/skills/Interceptor/SKILL.md b/LifeOS/install/skills/Interceptor/SKILL.md index 6ab3d8dff5..355e09d4f5 100644 --- a/LifeOS/install/skills/Interceptor/SKILL.md +++ b/LifeOS/install/skills/Interceptor/SKILL.md @@ -59,7 +59,7 @@ Each row is an independent capability class. Each one uses a different WebSocket | Verb tree | Top-level verbs | What you get | |-----------|-----------------|--------------| -| **VISUAL** | `screenshot` (DOM-render default — works backgrounded), `screenshot --region`, `screenshot --pixel --full` (window must be visible) | PNG/WebP at any size, selector, region, or scroll-and-stitch full page. Route through `Tools/Capture.sh`, never raw `interceptor screenshot`. | +| **VISUAL** | `screenshot` (DOM-render default — works backgrounded), `screenshot --region`, `screenshot --pixel --full` (window must be visible) | PNG/WebP at any size, selector, region, or scroll-and-stitch full page. Route through `Tools/Capture.sh`, never raw `interceptor screenshot`. Zoom into fine detail with `Tools/Zoom.ts`. | | **DOM READ** | `read [--markdown]`, `tree`, `text`, `html `, `find` | Accessibility tree, structured markdown, raw markup, refs | | **JS EVAL** | `eval `, `eval --main` | Run JavaScript in isolated or main world — **the way you read console errors, runtime exceptions, hydration warnings, and DOM state at runtime** | | **NETWORK** | `net log`, `net headers`, `net export --format har\|pcapng\|json`, `override`, `headers add/remove` | Passive request capture with zero CDP fingerprint; HAR 1.2 + pcapng for Wireshark | @@ -386,6 +386,7 @@ Agent(subagent_type="general-purpose", prompt=" The agent runs the `.app`-bundle binary (`~/.local/share/interceptor/interceptor-bridge.app/Contents/MacOS/interceptor-bridge`), NOT `/usr/local/bin/interceptor-bridge` (that copy is stale). A SIGKILLed bridge with a stale ad-hoc signature restart-loops every ~5s (`ThrottleInterval`) — re-sign the `.app` MacOS binary, not the `/usr/local/bin` copy. `Tools/HealBridge.sh` wraps the loaded-but-dead detect + single kickstart. *(2026-06-17.)* - **`AXEnhancedUserInterface` was removed from the bridge** (it foregrounded AppKit apps as a side effect of being interpreted as "VoiceOver active"). The bridge uses `AXManualAccessibility` exclusively for Electron wake-up. Stale guidance from old code that sets `AXEnhancedUserInterface` should be ignored. - **`screenshot --pixel --tab ` can capture the WRONG page — `--pixel` follows the active tab.** `--pixel` is `captureVisibleTab`; it shoots whatever tab is visually active, and if the preceding tab activation didn't land, that's the operator's foreground tab, not your target (observed 2026-06-11: requested a localhost site under test, captured the operator's unrelated foreground site instead). Always Read the returned image and confirm it's the page you asked for before citing it as evidence — this is why `Tools/Capture.sh` reads back on the `--pixel` fallback path. Do NOT reach for AppleScript tab activation — banned under Hard Prohibitions. Instead answer through a non-visual verb tree (`read --markdown`, `eval --main`, `net log`), or follow the WebSocket-wedge recovery ladder above. *(2026-06-11; AppleScript workaround removed 2026-06-13.)* +- **A full-page screenshot cannot settle an appearance claim about fine detail — zoom first.** Images are downscaled to roughly 1568px on the long edge before the model sees them, so on a 2000px-wide page a 40px logo arrives as a smudge. Cropping the delivered screenshot does not recover it; the detail was destroyed upstream. `bun Tools/Zoom.ts --x N --y N --w N --h N` crops from the full-resolution original and magnifies that region to the budget, so the pixels land on the thing under test. Reach for it whenever the claim is about a logo, glyph, font rendering, icon, or spacing of a few pixels — the class of claim behind the wrong-logo-3× incident that produced the "appearance ≠ existence" rule. Coordinates are absolute pixels in the source image; a DOM-coordinate read (`eval`) gives you the region to pass. Backends: `magick`, else `ffmpeg`. - **Sensitive-app gate.** The bridge rejects `type`/`keys`/`click x,y`/`drag` when frontmost is a denylisted bundle (Keychain, 1Password, Dashlane, LastPass, Bitwarden, System Settings, Chase, Bank of America, Wells Fargo). Surface the rejection — do not bypass. - **VM `paused-state` snapshots are gated.** `--paused-state` requires `validateSaveRestoreSupport()` to return true on the VM config. macOS guests support it; some Linux configurations don't. `--disk-only` always works. - **TCC grants for VM hosts:** the bridge needs `com.apple.security.virtualization` entitlement. Move the bridge out of `~/Documents` or `~/Desktop` if `setup_required` complains about the install location. diff --git a/LifeOS/install/skills/Interceptor/Tools/Zoom.ts b/LifeOS/install/skills/Interceptor/Tools/Zoom.ts new file mode 100755 index 0000000000..5a8bf485fc --- /dev/null +++ b/LifeOS/install/skills/Interceptor/Tools/Zoom.ts @@ -0,0 +1,148 @@ +#!/usr/bin/env bun +/** + * Zoom.ts — measured zoom for appearance verification. + * + * A full-page screenshot is downscaled to the model's image budget before it + * reaches the model, so fine detail — a logo, a glyph, 2px of spacing — is + * destroyed before anyone can look at it. Cropping the already-downscaled + * image does not bring it back. + * + * This crops the region from the FULL-RESOLUTION original and scales that + * region UP to fill the budget, so the pixels spent land on the thing being + * verified. Coordinates are absolute pixels in the original image. + * + * Usage: + * bun Zoom.ts --x N --y N --w N --h N [--out ] [--budget 1568] + * + * Exit 0 = wrote the zoom (path on stdout); 2 = bad args or out-of-bounds crop; + * 3 = no usable image backend. + * + * Backends, in preference order: `magick` (the same binary Capture.sh and + * VerifyImageProbe.ts use), then `ffmpeg`. Both produce the same crop-then- + * upscale; ffmpeg exists so the tool still works on hosts without ImageMagick. + */ +import { existsSync } from "fs"; + +/** Claude downscales images to ~1568px on the long edge; upscaling past that + * buys no detail and just costs tokens. */ +const DEFAULT_BUDGET = 1568; + +function magickBin(): string | null { + const w = Bun.which("magick"); + if (w) return w; + for (const p of ["/opt/homebrew/bin/magick", "/usr/local/bin/magick", "/usr/bin/magick"]) { + if (existsSync(p)) return p; + } + return null; +} + +function ffmpegBin(): string | null { + const w = Bun.which("ffmpeg"); + if (w) return w; + for (const p of ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"]) { + if (existsSync(p)) return p; + } + return null; +} + +function arg(name: string): string | undefined { + const i = process.argv.indexOf(`--${name}`); + return i === -1 ? undefined : process.argv[i + 1]; +} + +function die(code: number, msg: string): never { + console.error(msg); + process.exit(code); +} + +/** Source dimensions, via whichever backend is present. */ +function dimensions(path: string): { w: number; h: number } | null { + const m = magickBin(); + if (m) { + const r = Bun.spawnSync([m, path, "-format", "%w %h", "info:"]); + const [w, h] = r.stdout.toString().trim().split(/\s+/).map(Number); + if (Number.isFinite(w) && Number.isFinite(h)) return { w, h }; + } + const f = ffmpegBin(); + if (f) { + // ffmpeg reports the stream dimensions on stderr as "…, WxH[ …]" + const r = Bun.spawnSync([f, "-hide_banner", "-i", path, "-f", "null", "-"]); + const m2 = r.stderr.toString().match(/,\s(\d+)x(\d+)[\s,]/); + if (m2) return { w: Number(m2[1]), h: Number(m2[2]) }; + } + return null; +} + +const src = process.argv[2]; +if (!src || src.startsWith("--")) { + die(2, "usage: bun Zoom.ts --x N --y N --w N --h N [--out ] [--budget 1568]"); +} +if (!existsSync(src)) die(2, `no such image: ${src}`); + +const x = Number(arg("x")); +const y = Number(arg("y")); +const w = Number(arg("w")); +const h = Number(arg("h")); +if (![x, y, w, h].every(Number.isFinite)) { + die(2, "--x, --y, --w and --h are required and must be numbers (original-image pixels)"); +} +if (w <= 0 || h <= 0) die(2, `crop must have positive area, got ${w}x${h}`); +if (x < 0 || y < 0) die(2, `crop origin must be non-negative, got ${x},${y}`); + +const budget = Number(arg("budget") ?? DEFAULT_BUDGET); +if (!Number.isFinite(budget) || budget <= 0) die(2, `--budget must be a positive number, got ${arg("budget")}`); + +const dim = dimensions(src); +if (!dim) die(3, "no usable image backend — install ImageMagick (`magick`) or ffmpeg"); +if (x + w > dim.w || y + h > dim.h) { + die(2, `crop ${w}x${h}+${x}+${y} falls outside the ${dim.w}x${dim.h} original — coordinates are absolute pixels in the source image`); +} + +const out = arg("out") ?? src.replace(/(\.[a-z0-9]+)$/i, "") + `.zoom-${x}_${y}_${w}_${h}.png`; + +// Scale so the crop's LONG edge lands on the budget. Never downscale: if the +// region is already larger than the budget the model would shrink it anyway, +// and doing it here would just throw away detail earlier. +const longEdge = Math.max(w, h); +const scale = Math.max(1, budget / longEdge); +const outW = Math.round(w * scale); +const outH = Math.round(h * scale); + +const m = magickBin(); +const f = ffmpegBin(); +let ok = false; +let backend = ""; + +if (m) { + backend = "magick"; + const r = Bun.spawnSync([ + m, src, + "-crop", `${w}x${h}+${x}+${y}`, + "+repage", + "-filter", "point", // nearest-neighbour: magnify without inventing detail + "-resize", `${outW}x${outH}!`, + out, + ]); + ok = r.exitCode === 0; + if (!ok && !f) die(1, `magick failed: ${r.stderr.toString().trim()}`); +} + +if (!ok && f) { + backend = "ffmpeg"; + const r = Bun.spawnSync([ + f, "-hide_banner", "-loglevel", "error", "-y", + "-i", src, + "-vf", `crop=${w}:${h}:${x}:${y},scale=${outW}:${outH}:flags=neighbor`, + "-frames:v", "1", + out, + ]); + ok = r.exitCode === 0; + if (!ok) die(1, `ffmpeg failed: ${r.stderr.toString().trim()}`); +} + +if (!ok) die(3, "no usable image backend — install ImageMagick (`magick`) or ffmpeg"); + +console.error( + `zoom via ${backend}: ${w}x${h}+${x}+${y} of ${dim.w}x${dim.h} → ${outW}x${outH} (${scale.toFixed(2)}x)` +); +console.log(out);