feat(agent): wire automate/ax_interact computer tools (3/8 of #3307)

app/src-tauri/src/lib.rs

@@ -2822,7 +2822,16 @@ pub fn run() {
                             let run = req.run;
                             input_app
                                 .run_on_main_thread(move || {
-                                    let _ = tx.send((run)());
+                                    // Catch an enigo FFI panic so it can't unwind
+                                    // across the app main thread (which would be
+                                    // UB / abort). Convert it to a clean Err.
+                                    let result = std::panic::catch_unwind(
+                                        std::panic::AssertUnwindSafe(run),
+                                    )
+                                    .unwrap_or_else(|_| {
+                                        Err("synthetic input panicked on the main thread".to_string())
+                                    });
+                                    let _ = tx.send(result);
                                 })
                                 .map_err(|e| format!("run_on_main_thread dispatch failed: {e}"))?;
                             rx.await

app/src/utils/toolDefinitions.ts

@@ -45,6 +45,15 @@ export const TOOL_CATALOG: ToolDefinition[] = [
     defaultEnabled: true,
     rustToolNames: ['ax_interact'],
   },
+  {
+    id: 'automate',
+    displayName: 'App Automation',
+    description:
+      'Accomplish a multi-step goal in an app in one go (e.g. "play a song in Music", "message someone in Slack") — the agent drives the UI step by step.',
+    category: 'System',
+    defaultEnabled: true,
+    rustToolNames: ['automate'],
+  },
   {
     id: 'git_operations',
     displayName: 'Git Operations',

src/openhuman/agent_registry/agents/orchestrator/agent.toml

@@ -130,6 +130,23 @@ named = [
     # `computer_control.ax_interact_mutations`, and refuse a sensitive-app
     # denylist (password managers, Keychain, System Settings, terminals).
     "ax_interact",
+    # Multi-step UI automation in one call (e.g. "play <song> in Music",
+    # "message <person> in Slack"). Prefer over many individual ax_interact
+    # calls when the task needs several UI steps — a Rust perceive→act→verify
+    # loop runs the flow with a fast model. Same opt-in + sensitive-app denylist
+    # as ax_interact; `Dangerous`, gates through the ApprovalGate.
+    "automate",
+    # Full computer control (autonomy). Fallback for apps the accessibility API
+    # can't drive — notably Electron apps (Slack, Discord, VS Code) whose AX/UIA
+    # tree is empty. `screenshot` to see the screen, then `mouse` (move/click/
+    # drag/scroll) + `keyboard` (type/press/hotkey) to act by pixel coordinates.
+    # All `Dangerous` and gate through the ApprovalGate. mouse/keyboard require
+    # `computer_control.enabled = true`. Prefer `automate`/`ax_interact` first;
+    # use these when the AX tree comes back empty. NB: foreground the target app
+    # before typing/clicking (synthetic input goes to the focused window).
+    "screenshot",
+    "mouse",
+    "keyboard",
     # Time + scheduling — lets the orchestrator answer "what time is it",
     # "remind me in 10 minutes", "every morning at 8" directly rather than
     # delegating or telling the user it can't. `current_time` grounds

src/openhuman/agent_registry/agents/orchestrator/prompt.md

@@ -38,6 +38,25 @@ Follow this sequence for every user message:
 
 Default bias: **do not spawn a sub-agent when a direct response or direct tool call is sufficient** — but a live external-service request is *not* something to answer from memory, it requires the integration. Use `spawn_worker_thread` for long tasks that need their own thread.
 
+## Controlling desktop apps (full autonomy)
+
+You can open and operate native apps on this machine. **Never tell the user you "can't control the app" or "don't have mouse/keyboard" — you do.**
+
+**Rule 0 — foreground first, every time.** Before *any* keyboard/mouse action, call `launch_app "<App>"` for the target. `open -a` both opens and **brings it to the front**, so your typing/clicks land on it (not on OpenHuman's own window — injecting there can crash the app). Re-call `launch_app` right before each keyboard/mouse step if focus might have moved.
+
+**The reliable path is the keyboard, not the mouse.** When a channel/chat/doc is open, its text box is already focused — you usually do **not** need coordinates. Prefer this:
+
+1. `launch_app "<App>"` (foreground).
+2. `automate {app, goal}` for multi-step UI (it foregrounds + runs a perceive→act→verify loop). Good for native apps (Music, Mail, Notes).
+3. **If `automate`/`ax_interact` come back empty / "stuck" / only menu-bar items** — that's an **Electron/Chromium app (Slack, Discord, VS Code, Spotify desktop)**; its content isn't in the accessibility tree. Switch to **keyboard-driven control**:
+   - `launch_app "<App>"` (foreground), then `keyboard` `type` the text and `press` `Enter`. The focused input receives it. Use app **hotkeys** to navigate (no mouse needed).
+4. **Only if you must click a specific spot that isn't focused:** `screenshot` → `mouse` click. (Screenshots are downscaled so you can see them; coordinates you read are in the returned image's pixels.)
+
+**Worked example — "message hi on Slack" (keyboard-only, no vision):**
+`launch_app "Slack"` → `keyboard hotkey "cmd+k"` (Slack quick switcher) → `keyboard type "<person or channel>"` → `keyboard press "Enter"` (opens the chat, focuses the message box) → `keyboard type "hi"` → `keyboard press "Enter"` (sends). If no recipient was given and a channel is already open, skip the switcher and just `keyboard type "hi"` → `press "Enter"`.
+
+`screenshot`/`mouse`/`keyboard` run without an approval prompt (they're on your auto-approve list) — just proceed.
+
 ## Rules
 
 - **You are the chat tier.** You run on a fast UX-focused model (TTFT > deep reasoning). When a task needs sustained multi-step thinking — planning across many steps, comparing several non-obvious options, untangling ambiguous requirements — **delegate to the reasoning tier (`delegate_plan`)** rather than reasoning through it yourself. Your job at that point is to brief the planner well and synthesise its output back to the user.

src/openhuman/tools/impl/computer/automate.rs

@@ -0,0 +1,223 @@
+//! Tool: `automate` — accomplish a multi-step UI goal in one call.
+//!
+//! The orchestrator calls `automate{app, goal}` once; the Rust loop in
+//! `accessibility::automate` then perceives → decides (fast model) → acts →
+//! settles → verifies until the goal is met or a step budget is hit. This keeps
+//! the heavy chat model out of the click loop (latency + reliability — see
+//! `docs/voice-automate-plan.md`).
+//!
+//! Safety mirrors `ax_interact`: it actuates real controls, so it is a mutating
+//! tool — opt-in via `computer_control.ax_interact_mutations`, routed through the
+//! ApprovalGate, and it refuses the sensitive-app denylist (password managers,
+//! Keychain, System Settings, terminals) even on auto-approved turns.
+
+use super::ax_interact::is_sensitive_app;
+use crate::openhuman::accessibility::automate::{self, AutomateOptions, RealBackend};
+use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolCallOptions, ToolResult};
+use async_trait::async_trait;
+use serde_json::json;
+
+pub struct AutomateTool {
+    /// When false the tool refuses to run (it is inherently mutating). Mirrors
+    /// `AxInteractTool::allow_mutations` so one opt-in governs both.
+    allow_mutations: bool,
+}
+
+impl AutomateTool {
+    pub fn new(allow_mutations: bool) -> Self {
+        Self { allow_mutations }
+    }
+}
+
+impl Default for AutomateTool {
+    fn default() -> Self {
+        Self::new(false)
+    }
+}
+
+#[async_trait]
+impl Tool for AutomateTool {
+    fn name(&self) -> &str {
+        "automate"
+    }
+
+    fn description(&self) -> &str {
+        "Accomplish a MULTI-STEP goal inside a desktop app in a single call — e.g. \
+         'play <song> in Music', 'message <person> <text> in Slack'. Give the app \
+         name and a plain-English goal; the system drives the app's UI step by step \
+         (find elements → press/type → verify) using the platform accessibility API, \
+         no screen coordinates. Prefer this over issuing many individual \
+         `ax_interact` calls when the task needs several UI steps. The app should \
+         usually be launched first (or include 'launch' in the goal). Refuses \
+         password managers, Keychain, System Settings, and terminals."
+    }
+
+    fn parameters_schema(&self) -> serde_json::Value {
+        json!({
+            "type": "object",
+            "properties": {
+                "app": {
+                    "type": "string",
+                    "description": "Display name of the target application (e.g. 'Music', 'Slack')."
+                },
+                "goal": {
+                    "type": "string",
+                    "description": "Plain-English description of the multi-step outcome to achieve."
+                }
+            },
+            "required": ["app", "goal"]
+        })
+    }
+
+    fn permission_level(&self) -> PermissionLevel {
+        // Always mutating — it actuates controls. Kept as the base level so the
+        // approval gate fires regardless of args.
+        PermissionLevel::Dangerous
+    }
+
+    fn external_effect(&self) -> bool {
+        true
+    }
+
+    async fn execute(&self, args: serde_json::Value) -> anyhow::Result<ToolResult> {
+        self.execute_with_options(args, ToolCallOptions::default())
+            .await
+    }
+
+    async fn execute_with_options(
+        &self,
+        args: serde_json::Value,
+        _options: ToolCallOptions,
+    ) -> anyhow::Result<ToolResult> {
+        let app = args
+            .get("app")
+            .and_then(|v| v.as_str())
+            .unwrap_or("")
+            .trim()
+            .to_string();
+        let goal = args
+            .get("goal")
+            .and_then(|v| v.as_str())
+            .unwrap_or("")
+            .trim()
+            .to_string();
+
+        log::info!("[automate] ▶ tool execute app={app:?} goal={goal:?}");
+
+        if app.is_empty() {
+            return Ok(ToolResult::error("app is required"));
+        }
+        if goal.is_empty() {
+            return Ok(ToolResult::error("goal is required"));
+        }
+
+        // Hard safety boundary — identical to ax_interact's denylist.
+        if is_sensitive_app(&app) {
+            log::warn!("[automate] refused: sensitive app '{app}'");
+            return Ok(ToolResult::error(format!(
+                "Refusing to automate '{app}': it is on the sensitive-app denylist \
+                 (password managers, Keychain, System Settings, terminals). This is a \
+                 hard safety boundary."
+            )));
+        }
+
+        if !self.allow_mutations {
+            log::warn!("[automate] refused: mutations disabled");
+            return Ok(ToolResult::error(
+                "App control isn't enabled yet. Turn on App Automation in \
+                 Settings → Agent Access (it grants permission to control apps), \
+                 then ask again. (Sets computer_control.ax_interact_mutations = true.)",
+            ));
+        }
+
+        let config = match crate::openhuman::config::rpc::load_config_with_timeout().await {
+            Ok(c) => c,
+            Err(e) => return Ok(ToolResult::error(format!("could not load config: {e}"))),
+        };
+
+        let backend = RealBackend::new(config);
+        let outcome = automate::run(&app, &goal, &backend, AutomateOptions::default()).await;
+
+        let mut body = format!("{}\n\nSteps:", outcome.summary);
+        if outcome.steps.is_empty() {
+            body.push_str("\n  (no steps executed)");
+        } else {
+            for s in &outcome.steps {
+                body.push_str(&format!("\n  - {s}"));
+            }
+        }
+
+        if outcome.success {
+            Ok(ToolResult::success(body))
+        } else {
+            Ok(ToolResult::error(body))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn name_and_permission() {
+        let t = AutomateTool::new(true);
+        assert_eq!(t.name(), "automate");
+        assert_eq!(t.permission_level(), PermissionLevel::Dangerous);
+        assert!(t.external_effect());
+    }
+
+    #[test]
+    fn schema_requires_app_and_goal() {
+        let schema = AutomateTool::new(true).parameters_schema();
+        let req = schema["required"].as_array().unwrap();
+        assert!(req.iter().any(|v| v == "app"));
+        assert!(req.iter().any(|v| v == "goal"));
+    }
+
+    #[tokio::test]
+    async fn rejects_missing_app_or_goal() {
+        let t = AutomateTool::new(true);
+        assert!(
+            t.execute(json!({"app": "", "goal": "x"}))
+                .await
+                .unwrap()
+                .is_error
+        );
+        assert!(
+            t.execute(json!({"app": "Music", "goal": ""}))
+                .await
+                .unwrap()
+                .is_error
+        );
+    }
+
+    #[tokio::test]
+    async fn refuses_when_mutations_disabled() {
+        let t = AutomateTool::new(false);
+        let r = t
+            .execute(json!({"app": "Music", "goal": "play a song"}))
+            .await
+            .unwrap();
+        assert!(r.is_error);
+        assert!(r.output().contains("ax_interact_mutations"));
+    }
+
+    #[tokio::test]
+    async fn refuses_sensitive_app() {
+        let t = AutomateTool::new(true);
+        for app in [
+            "Keychain Access",
+            "1Password",
+            "Terminal",
+            "System Settings",
+        ] {
+            let r = t
+                .execute(json!({"app": app, "goal": "do something"}))
+                .await
+                .unwrap();
+            assert!(r.is_error, "expected refusal for {app}");
+            assert!(r.output().to_lowercase().contains("denylist"));
+        }
+    }
+}

src/openhuman/tools/impl/computer/ax_interact.rs

@@ -48,16 +48,20 @@ const SENSITIVE_APPS: &[&str] = &[
     "rio",
 ];
 
-fn is_sensitive_app(app_name: &str) -> bool {
+/// True when `app_name` is on the never-actuate denylist. `pub(crate)` so the
+/// `automate` tool shares the exact same boundary as `ax_interact`.
+pub(crate) fn is_sensitive_app(app_name: &str) -> bool {
     let lower = app_name.to_lowercase();
     SENSITIVE_APPS.iter().any(|s| lower.contains(s))
 }
 
 pub struct AxInteractTool {
     /// When false, the mutating actions (`press` / `set_value`) are refused
     /// with guidance to enable `computer_control.ax_interact_mutations`. The
-    /// read-only `list` action is always available. Mirrors the opt-in posture
-    /// of the mouse/keyboard tools (`computer_control.enabled`).
+    /// read-only `list` action is always available. Like the mouse/keyboard
+    /// tools (`computer_control.enabled`), this is opt-in **and** approval-gated:
+    /// the mutating actions return `external_effect_with_args == true` so they
+    /// route through the ApprovalGate.
     allow_mutations: bool,
 }
 
@@ -229,10 +233,10 @@ impl Tool for AxInteractTool {
         if mutating && !self.allow_mutations {
             log::warn!("[ax_interact] refused: mutations disabled (action={action})");
             return Ok(ToolResult::error(
-                "ax_interact mutations (press/set_value) are disabled. They actuate arbitrary \
-                 app controls and type into arbitrary fields, so they require explicit opt-in: \
-                 set `computer_control.ax_interact_mutations = true`. The read-only 'list' \
-                 action remains available.",
+                "App control isn't enabled yet, so I can't press buttons or type into \
+                 this app. Turn on App UI Control / App Automation in Settings → Agent \
+                 Access, then ask again. (Reading the UI still works without it; sets \
+                 computer_control.ax_interact_mutations = true.)",
             ));
         }
 

src/openhuman/tools/impl/computer/keyboard.rs

@@ -115,6 +115,16 @@ impl Tool for KeyboardTool {
         PermissionLevel::Dangerous
     }
 
+    /// Route every call through the ApprovalGate. Arbitrary keystrokes can land
+    /// in a focused sudo/password field or Terminal, and there's no sensitive-app
+    /// denylist for raw input, so the gate is the only boundary — and it fires on
+    /// `external_effect_with_args`, NOT on `PermissionLevel::Dangerous`. Without
+    /// this, keystrokes could run unattended on an auto-approved turn once
+    /// `computer_control.enabled`.
+    fn external_effect(&self) -> bool {
+        true
+    }
+
     fn parameters_schema(&self) -> Value {
         json!({
             "type": "object",