Merge pull request #210 from LockInTime/feat/35-semantic-hover

feat: add bounded semantic hover

Author: yashranaway

.agents/skills/headless-computer-use/SKILL.md

@@ -32,6 +32,8 @@ Chromium capabilities; macOS WebKit reports them as unsupported.
 4. Start with `inspect --context summary --task "..."`, then use an outline,
    scoped text, or scoped actions only when the task needs them.
 5. Prefer role/name targeting; otherwise use a ref from the latest inspection.
+   Use `hover` when the task requires a tooltip or other hover-only state; it
+   does not click or focus the target. Inspect again after the state changes.
    For native dropdowns, use `select` with exactly one exact `--label` or
    `--value`; do not treat custom ARIA widgets as native selects.
 6. After navigation or a substantial rerender, wait for the expected URL, text,
@@ -49,6 +51,7 @@ headless session create agent-qa
 headless --session agent-qa visit http://localhost:3000
 headless --session agent-qa inspect --context summary --task "click Continue"
 headless --session agent-qa inspect --context actions --task "click Continue" --limit 10
+headless --session agent-qa hover --role button --name Continue
 headless --session agent-qa click --role button --name Continue
 headless --session agent-qa wait --url /next --settled --timeout 10000
 headless --session agent-qa inspect --context actions --task "verify next page"

.agents/skills/headless-computer-use/references/commands.md

@@ -28,6 +28,8 @@ headless --session NAME inspect --context actions --task "TASK"
 headless --session NAME inspect --context full --text
 headless --session NAME click REF
 headless --session NAME click --role ROLE --name NAME
+headless --session NAME hover REF
+headless --session NAME hover --role ROLE --name NAME
 headless --session NAME fill REF "TEXT"
 headless --session NAME fill REF -- "--json  stays literal"
 headless --session NAME select REF --label LABEL
@@ -48,7 +50,9 @@ large pages, request `outline`, select a returned `@rN` region, then use
 bound the result; check `omitted` before assuming it describes the whole page.
 Use `click --role ... --name ...` for unique accessible controls. Use an `@eN`
 ref from the latest inspection when role/name is ambiguous. Inspect again after
-navigation or a large rerender. Native single-select controls advertise
+navigation or a large rerender. Use `hover` only for hover-dependent state such
+as tooltips; it does not focus or click, and its output never includes internal
+coordinates. Native single-select controls advertise
 `select`; use exactly one exact `--label` or `--value`. File inputs advertise
 `upload` for an existing private artifact-store basename. Upload never accepts
 or imports a filesystem path. Ask before uploading, as in [safety.md](safety.md).

CHANGELOG.md

@@ -12,8 +12,8 @@ Two versions travel independently, on purpose:
   `HEADLESS_VERSION`, reported by the CLI, host, and MCP adapter, and used for
   release assets.
 - **Protocol version**: `headlessProtocolVersion` in `Protocol.swift`,
-  currently `0.5`. It changes only when the wire contract changes, and always
-  with an entry in
+  currently `0.5`. It changes only for incompatible wire-contract changes,
+  always with an entry in
   [`docs/roadmap/architecture-decisions.md`](docs/roadmap/architecture-decisions.md).
 
 ## [Unreleased]
@@ -24,6 +24,9 @@ Two versions travel independently, on purpose:
 
 ### Added
 
+- Semantic `hover` targets a fresh element ref or an exact role/name without
+  exposing coordinates. Chromium dispatches a trusted CDP mouse move, while
+  WebKit reports its fixed synthetic hover path in capabilities.
 - macOS Command-, opens one Settings window built from the typed registry and
   writing through the same backend as `headless config`. Linux has no Settings
   GUI.

README.md

@@ -79,6 +79,7 @@ headless --session qa inspect --context outline --limit 20
 headless --session qa inspect --context actions --task "click Continue"
 headless --session qa record start --fps 10
 headless --session qa tour --full-page
+headless --session qa hover --role button --name Continue
 headless --session qa click --role button --name Continue
 headless --session qa wait --url /next --settled
 headless --session qa record stop --output dashboard-flow.mp4
@@ -128,7 +129,10 @@ structural region references such as `@r4`, then inspect only that region with
 explicit broad-page escape hatch.
 
 Each control's `actions` list contains only protocol verbs that can run (`click`,
-`fill`, or `upload`); unsupported controls never advertise nonexistent commands. Full
+`fill`, `select`, or `upload`); unsupported controls never advertise nonexistent
+commands. Hover is available for any visible semantic target but is not
+advertised because a page does not expose whether hovering has meaningful
+behavior. Full
 inspection retains roles, names, rendered media metadata, safety markers,
 bounds, and element references such as `@e1`.
 `capture-info` returns the browser surface, page state, action trace, and

apps/headless-rs/src/protocol.rs

@@ -26,6 +26,7 @@ pub enum CommandName {
     Visit,
     Inspect,
     Click,
+    Hover,
     Fill,
     Press,
     Scroll,
@@ -93,6 +94,7 @@ impl CommandName {
             CommandName::Visit => "visit",
             CommandName::Inspect => "inspect",
             CommandName::Click => "click",
+            CommandName::Hover => "hover",
             CommandName::Fill => "fill",
             CommandName::Press => "press",
             CommandName::Scroll => "scroll",
@@ -350,7 +352,7 @@ impl CommandRequest {
                 }
                 Ok(())
             }
-            Click => self.target(false, true, true),
+            Click | Hover => self.target(false, true, true),
             Fill => self.target(true, true, true),
             Press => {
                 self.allow(&["key"])?;

apps/headless-rs/tests/protocol_tests.rs

@@ -165,6 +165,22 @@ fn command_parameter_validation() {
     let request = valid_request(CommandName::Click, params(&[("target", json!("@food"))]));
     assert!(request.validate().is_err());
 
+    // hover follows the same exclusive target contract as click
+    let request = valid_request(CommandName::Hover, params(&[("target", json!("@e8"))]));
+    request.validate().unwrap();
+    let request = valid_request(
+        CommandName::Hover,
+        params(&[("role", json!("button")), ("name", json!("Account"))]),
+    );
+    request.validate().unwrap();
+    let request = valid_request(CommandName::Hover, params(&[]));
+    assert!(request.validate().is_err());
+    let request = valid_request(
+        CommandName::Hover,
+        params(&[("target", json!("@e8")), ("name", json!("Account"))]),
+    );
+    assert!(request.validate().is_err());
+
     // fill requires a value
     let request = valid_request(CommandName::Fill, params(&[("target", json!("@e12"))]));
     assert!(request.validate().is_err());

apps/headless/Host/AgentBridge.swift

@@ -57,6 +57,11 @@ extension BrowserWindowController {
         return try callAgent("return globalThis.__headlessAgent.click(args);", arguments: ["args": args])
     }
 
+    func agentHover(parameters: [String: JSONValue]) throws -> JSONValue {
+        let args = try browserTargetArguments(parameters)
+        return try callAgent("return globalThis.__headlessAgent.hover(args);", arguments: ["args": args])
+    }
+
     func agentFill(parameters: [String: JSONValue]) throws -> JSONValue {
         var args = try browserTargetArguments(parameters)
         guard let value = parameters["value"]?.stringValue else {
@@ -644,6 +649,9 @@ extension BrowserWindowController: BrowserEngineSession {
     func hostClick(parameters: [String: JSONValue]) throws -> JSONValue {
         try agentClick(parameters: parameters)
     }
+    func hostHover(parameters: [String: JSONValue]) throws -> JSONValue {
+        try agentHover(parameters: parameters)
+    }
     func hostFill(parameters: [String: JSONValue]) throws -> JSONValue {
         try agentFill(parameters: parameters)
     }

apps/headless/LinuxHost/BrowserProcess.swift

@@ -773,6 +773,21 @@ final class LinuxBrowserSession: @unchecked Sendable {
         ])
     }
 
+    func hover(parameters: [String: JSONValue]) throws -> JSONValue {
+        let target = try trustedInputTarget(parameters: parameters, action: "hover")
+        _ = try command("Input.dispatchMouseEvent", parameters: [
+            "type": "mouseMoved", "x": target.x, "y": target.y,
+        ])
+        // A hover handler can start a cross-document navigation before the next
+        // command. Pause capture so a recording does not composite that frame.
+        pauseRecordingCapture()
+        return .object([
+            "hovered": .string(target.reference),
+            "role": .string(target.role),
+            "name": .string(target.name),
+        ])
+    }
+
     func upload(parameters: [String: JSONValue], artifactURL: URL) throws -> JSONValue {
         let args = try browserTargetArguments(parameters)
         let objectId = try evaluateNode(

apps/headless/LinuxHost/main.swift

@@ -88,6 +88,9 @@ final class ChromiumBrowserEngineSession: BrowserEngineSession {
     func hostClick(parameters: [String: JSONValue]) throws -> JSONValue {
         try browserSession.click(parameters: parameters)
     }
+    func hostHover(parameters: [String: JSONValue]) throws -> JSONValue {
+        try browserSession.hover(parameters: parameters)
+    }
     func hostFill(parameters: [String: JSONValue]) throws -> JSONValue {
         try browserSession.fill(parameters: parameters)
     }

apps/headless/Sources/HeadlessProtocol/CLI.swift

@@ -151,6 +151,8 @@ public struct CLIParser {
             return try parseInspect(arguments, session: session, jsonOutput: jsonOutput)
         case "click":
             return try parseTargeted(.click, arguments: arguments, session: session, jsonOutput: jsonOutput)
+        case "hover":
+            return try parseTargeted(.hover, arguments: arguments, session: session, jsonOutput: jsonOutput)
         case "upload":
             return try parseUpload(arguments, session: session, jsonOutput: jsonOutput)
         case "fill":
@@ -884,6 +886,7 @@ Commands:
   inspect [--context summary|outline|text|actions|full] [--task TEXT]
           [--within @rN] [--limit N] [--budget TOKENS] [--depth N] [--text]
   click REF | click --role ROLE [--name NAME]
+  hover REF | hover --role ROLE [--name NAME]
   fill REF TEXT | fill REF -- TEXT_WITH_LITERAL_FLAGS | press KEY
   select REF --label LABEL | select REF --value VALUE
   select --role ROLE [--name NAME] (--label LABEL | --value VALUE)