feat(computer-use): update settings (#263)

Co-authored-by: bowen628 <bowen628@noreply.gitcode.com>

Author: bobleer

src/crates/core/src/agentic/agents/prompts/claw_mode.md

@@ -50,7 +50,7 @@ When the `ComputerUse` tool is available, you may capture the screen and use mou
 - **Strict rule — no blind Enter, no blind click:** Before **`click`**, you **must** have a **fine** screenshot after the pointer is aligned: **`quadrant_navigation_click_ready`: true** (preferred: **`screenshot` + `screenshot_navigate_quadrant`** each step until the tool JSON says so) **or** a **point-crop `screenshot`** (~500×500 via `screenshot_crop_center_*`) when the exceptions above apply. A **full-screen-only** frame alone does **not** authorize **`click`**. Before **`key_chord` that includes Return or Enter**, you **must** call **`screenshot` first** and **visually confirm** focus and target. The only exception is when the user explicitly asks for an unverified / blind step.
 - For sending messages, payments, destructive actions, or anything sensitive, state the exact steps first and obtain clear user confirmation in chat before executing.
 - If Computer use is disabled or OS permissions are missing, tell the user what to enable in BitFun settings / system privacy instead of claiming success.
-- Screenshot results require the session primary model to use Anthropic API format so the image is attached to the tool result for vision. The JPEG matches **native display resolution** (no downscale): `coordinate_mode` `"image"` uses the same pixel grid as the bitmap.
+- Screenshot results require the session primary model to use Anthropic or OpenAI-compatible API format so the image is attached to the tool result for vision. The JPEG matches **native display resolution** (no downscale): `coordinate_mode` `"image"` uses the same pixel grid as the bitmap.
 - **Host-enforced screenshot (two cases):** The desktop host **rejects `click`** until the last `screenshot` after the last pointer move is a **valid fine basis**: **`quadrant_navigation_click_ready`: true** (quadrant drill until the region’s longest side is below the host threshold) **or** a **fresh point-crop** (`screenshot_crop_center_*`, ~500×500). **Full-screen-only** is **not** enough. It **rejects `key_chord` that includes Return or Enter** until a **fresh `screenshot`** since the last pointer move or click. **`mouse_move`** may use **`coordinate_mode` `\"image\"`** on any prior **`screenshot`**. Still **prefer `key_chord`** when it matches the step.
 - **Rulers vs zoom:** Full-frame JPEGs have **margin rulers** and a **grid** — use them to orient. For small controls, **default to quadrant drill** (`screenshot_navigate_quadrant` on each `screenshot` step); use **point crop** only as a **secondary** option (see default path above). Each quadrant step **adds padding on every side** (clamped) so controls on split lines stay in the JPEG. **Do not** rely only on huge full-display images when a smaller view answers the question.
 - **Click guard:** The host **rejects `click`** if there was **`mouse_move` / `pointer_nudge` / `pointer_move_rel` or a previous `click`** since the last `screenshot`, or if the last `screenshot` was **full-screen only** without **`quadrant_navigation_click_ready`**. **`screenshot`** before **Return/Enter** in **`key_chord`** when the outcome matters.

src/crates/core/src/agentic/tools/implementations/computer_use_tool.rs

@@ -36,12 +36,18 @@ impl ComputerUseTool {
             .to_lowercase()
     }
 
-    fn require_anthropic_for_screenshot(ctx: &ToolUseContext) -> BitFunResult<()> {
-        if Self::primary_api_format(ctx) == "anthropic" {
+    /// Screenshot tool results attach JPEGs via `tool_image_attachments`; only providers whose
+    /// request converters emit multimodal tool output are supported (Anthropic + OpenAI-compatible).
+    fn require_multimodal_tool_output_for_screenshot(ctx: &ToolUseContext) -> BitFunResult<()> {
+        let f = Self::primary_api_format(ctx);
+        if matches!(
+            f.as_str(),
+            "anthropic" | "openai" | "response" | "responses"
+        ) {
             return Ok(());
         }
         Err(BitFunError::tool(
-            "Screenshot results include images in tool results; set the primary model to an Anthropic (Claude) API format. Other providers are not supported for screenshots yet.".to_string(),
+            "Screenshot results include images in tool results; set the primary model to Anthropic (Claude) or OpenAI-compatible API format. Other providers are not supported for screenshots yet.".to_string(),
         ))
     }
 
@@ -598,7 +604,7 @@ Each **`screenshot`** JPEG: **four-side margin coordinate scales** (numbers), **
 
         match action {
             "screenshot" => {
-                Self::require_anthropic_for_screenshot(context)?;
+                Self::require_multimodal_tool_output_for_screenshot(context)?;
                 let (params, ignored_crop_for_quadrant) = Self::parse_screenshot_params(input)?;
                 let crop_for_debug = params.crop_center;
                 let nav_debug = params.navigate_quadrant.map(|q| match q {

src/crates/core/src/infrastructure/ai/providers/openai/message_converter.rs

@@ -83,9 +83,37 @@ impl OpenAIMessageConverter {
 
     fn convert_tool_message_to_responses_item(msg: Message) -> Option<Value> {
         let call_id = msg.tool_call_id?;
-        let output = msg
-            .content
-            .unwrap_or_else(|| "Tool execution completed".to_string());
+        let text = msg.content.unwrap_or_default();
+
+        // Responses API: `output` may be a string or a list of input_text / input_image / input_file
+        // (see OpenAI FunctionCallOutput schema).
+        let output: Value = if let Some(attachments) = msg.tool_image_attachments.filter(|a| !a.is_empty()) {
+            let mut parts: Vec<Value> = attachments
+                .into_iter()
+                .map(|att| {
+                    let data_url = format!("data:{};base64,{}", att.mime_type, att.data_base64);
+                    json!({
+                        "type": "input_image",
+                        "image_url": data_url
+                    })
+                })
+                .collect();
+            parts.push(json!({
+                "type": "input_text",
+                "text": if text.is_empty() {
+                    "Tool execution completed".to_string()
+                } else {
+                    text
+                }
+            }));
+            json!(parts)
+        } else {
+            json!(if text.is_empty() {
+                "Tool execution completed".to_string()
+            } else {
+                text
+            })
+        };
 
         Some(json!({
             "type": "function_call_output",
@@ -171,6 +199,44 @@ impl OpenAIMessageConverter {
     }
 
     fn convert_single_message(msg: Message) -> Value {
+        // Chat Completions: multimodal tool message (e.g. GPT-4o vision + tools) — image parts + text.
+        if msg.role == "tool" {
+            if let Some(ref attachments) = msg.tool_image_attachments {
+                if !attachments.is_empty() {
+                    let mut parts: Vec<Value> = attachments
+                        .iter()
+                        .map(|att| {
+                            let url = format!("data:{};base64,{}", att.mime_type, att.data_base64);
+                            json!({
+                                "type": "image_url",
+                                "image_url": { "url": url, "detail": "auto" }
+                            })
+                        })
+                        .collect();
+                    let text = msg.content.clone().unwrap_or_default();
+                    if text.trim().is_empty() {
+                        parts.push(json!({
+                            "type": "text",
+                            "text": "Tool execution completed"
+                        }));
+                    } else {
+                        parts.push(json!({ "type": "text", "text": text }));
+                    }
+                    let mut openai_msg = json!({
+                        "role": "tool",
+                        "content": Value::Array(parts),
+                    });
+                    if let Some(id) = msg.tool_call_id {
+                        openai_msg["tool_call_id"] = Value::String(id);
+                    }
+                    if let Some(name) = msg.name {
+                        openai_msg["name"] = Value::String(name);
+                    }
+                    return openai_msg;
+                }
+            }
+        }
+
         let mut openai_msg = json!({
             "role": msg.role,
         });
@@ -282,7 +348,7 @@ impl OpenAIMessageConverter {
 #[cfg(test)]
 mod tests {
     use super::OpenAIMessageConverter;
-    use crate::util::types::{Message, ToolCall};
+    use crate::util::types::{Message, ToolCall, ToolImageAttachment};
     use serde_json::json;
     use std::collections::HashMap;
 
@@ -354,4 +420,57 @@ mod tests {
         assert_eq!(content[0]["type"], json!("input_image"));
         assert_eq!(content[1]["type"], json!("input_text"));
     }
+
+    #[test]
+    fn converts_tool_message_with_images_to_responses_function_call_output() {
+        let messages = vec![Message {
+            role: "tool".to_string(),
+            content: Some("Screen captured".to_string()),
+            reasoning_content: None,
+            thinking_signature: None,
+            tool_calls: None,
+            tool_call_id: Some("call_cu_1".to_string()),
+            name: Some("computer_use".to_string()),
+            tool_image_attachments: Some(vec![ToolImageAttachment {
+                mime_type: "image/jpeg".to_string(),
+                data_base64: "AAA".to_string(),
+            }]),
+        }];
+
+        let (_, input) = OpenAIMessageConverter::convert_messages_to_responses_input(messages);
+        let out = &input[0];
+        assert_eq!(out["type"], json!("function_call_output"));
+        assert_eq!(out["call_id"], json!("call_cu_1"));
+        let output = out["output"].as_array().expect("multimodal output");
+        assert_eq!(output[0]["type"], json!("input_image"));
+        assert!(output[0]["image_url"]
+            .as_str()
+            .unwrap()
+            .starts_with("data:image/jpeg;base64,"));
+        assert_eq!(output[1]["type"], json!("input_text"));
+        assert_eq!(output[1]["text"], json!("Screen captured"));
+    }
+
+    #[test]
+    fn converts_tool_message_with_images_to_chat_completions_content_parts() {
+        let msg = Message {
+            role: "tool".to_string(),
+            content: Some("ok".to_string()),
+            reasoning_content: None,
+            thinking_signature: None,
+            tool_calls: None,
+            tool_call_id: Some("call_1".to_string()),
+            name: Some("computer_use".to_string()),
+            tool_image_attachments: Some(vec![ToolImageAttachment {
+                mime_type: "image/jpeg".to_string(),
+                data_base64: "YmFi".to_string(),
+            }]),
+        };
+
+        let openai = OpenAIMessageConverter::convert_messages(vec![msg]);
+        let content = openai[0]["content"].as_array().expect("content parts");
+        assert_eq!(content[0]["type"], json!("image_url"));
+        assert_eq!(content[1]["type"], json!("text"));
+        assert_eq!(content[1]["text"], json!("ok"));
+    }
 }

src/web-ui/src/infrastructure/config/components/AIFeaturesConfig.scss

@@ -15,6 +15,17 @@
     gap: $size-gap-2;
   }
 
+  /** Computer use: authorized status label (Granted / 已授权) */
+  &__perm-status--granted {
+    color: var(--color-text-muted);
+  }
+
+  /** Computer use permission row: keep label short, avoid squashing action buttons (nowrap). */
+  &__row-action-btn {
+    flex-shrink: 0;
+    white-space: nowrap;
+  }
+
   &__row-control--model {
     width: 100%;
     align-items: stretch;

src/web-ui/src/infrastructure/config/components/SessionConfig.tsx

@@ -520,31 +520,95 @@ const SessionConfig: React.FC = () => {
                   <span className="bitfun-func-agent-config__hint">{computerUseNote}</span>
                 </ConfigPageRow>
               ) : null}
-              <ConfigPageRow label={t('computerUse.accessibility')} align="center">
-                <div className="bitfun-func-agent-config__row-control" style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
-                  <span>{computerUseAccess ? t('computerUse.granted') : t('computerUse.notGranted')}</span>
-                  <Button size="small" variant="secondary" disabled={computerUseBusy} onClick={() => void handleComputerUseRequestPermissions()}>
-                    {t('computerUse.request')}
-                  </Button>
-                  <Button size="small" variant="secondary" disabled={computerUseBusy} onClick={() => void handleComputerUseOpenSettings('accessibility')}>
+              <ConfigPageRow label={t('computerUse.accessibility')} align="center" balanced>
+                <div
+                  className="bitfun-func-agent-config__row-control"
+                  style={{
+                    display: 'flex',
+                    flexDirection: 'row',
+                    flexWrap: 'nowrap',
+                    alignItems: 'center',
+                    justifyContent: 'flex-end',
+                    gap: 8,
+                  }}
+                >
+                  <span style={{ display: 'inline-flex', alignItems: 'center', gap: 6, flexShrink: 0 }}>
+                    <span className={computerUseAccess ? 'bitfun-func-agent-config__perm-status--granted' : undefined}>
+                      {computerUseAccess ? t('computerUse.granted') : t('computerUse.notGranted')}
+                    </span>
+                    <IconButton
+                      type="button"
+                      size="small"
+                      variant="ghost"
+                      aria-label={t('computerUse.refreshStatus')}
+                      tooltip={t('computerUse.refreshStatus')}
+                      disabled={computerUseBusy}
+                      onClick={() => void refreshComputerUseStatus()}
+                    >
+                      <RefreshCw size={14} />
+                    </IconButton>
+                  </span>
+                  {!computerUseAccess ? (
+                    <Button
+                      className="bitfun-func-agent-config__row-action-btn"
+                      size="small"
+                      variant="secondary"
+                      disabled={computerUseBusy}
+                      onClick={() => void handleComputerUseRequestPermissions()}
+                    >
+                      {t('computerUse.request')}
+                    </Button>
+                  ) : null}
+                  <Button
+                    className="bitfun-func-agent-config__row-action-btn"
+                    size="small"
+                    variant="secondary"
+                    disabled={computerUseBusy}
+                    onClick={() => void handleComputerUseOpenSettings('accessibility')}
+                  >
                     {t('computerUse.openSettings')}
                   </Button>
                 </div>
               </ConfigPageRow>
-              <ConfigPageRow label={t('computerUse.screenCapture')} align="center">
-                <div className="bitfun-func-agent-config__row-control" style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
-                  <span>{computerUseScreen ? t('computerUse.granted') : t('computerUse.notGranted')}</span>
-                  <Button size="small" variant="secondary" disabled={computerUseBusy} onClick={() => void handleComputerUseOpenSettings('screen_capture')}>
+              <ConfigPageRow label={t('computerUse.screenCapture')} align="center" balanced>
+                <div
+                  className="bitfun-func-agent-config__row-control"
+                  style={{
+                    display: 'flex',
+                    flexDirection: 'row',
+                    flexWrap: 'nowrap',
+                    alignItems: 'center',
+                    justifyContent: 'flex-end',
+                    gap: 8,
+                  }}
+                >
+                  <span style={{ display: 'inline-flex', alignItems: 'center', gap: 6, flexShrink: 0 }}>
+                    <span className={computerUseScreen ? 'bitfun-func-agent-config__perm-status--granted' : undefined}>
+                      {computerUseScreen ? t('computerUse.granted') : t('computerUse.notGranted')}
+                    </span>
+                    <IconButton
+                      type="button"
+                      size="small"
+                      variant="ghost"
+                      aria-label={t('computerUse.refreshStatus')}
+                      tooltip={t('computerUse.refreshStatus')}
+                      disabled={computerUseBusy}
+                      onClick={() => void refreshComputerUseStatus()}
+                    >
+                      <RefreshCw size={14} />
+                    </IconButton>
+                  </span>
+                  <Button
+                    className="bitfun-func-agent-config__row-action-btn"
+                    size="small"
+                    variant="secondary"
+                    disabled={computerUseBusy}
+                    onClick={() => void handleComputerUseOpenSettings('screen_capture')}
+                  >
                     {t('computerUse.openSettings')}
                   </Button>
                 </div>
               </ConfigPageRow>
-              <ConfigPageRow label={t('computerUse.refreshStatus')} align="center">
-                <Button size="small" variant="secondary" disabled={computerUseBusy} onClick={() => void refreshComputerUseStatus()}>
-                  <RefreshCw size={14} style={{ marginRight: 6 }} />
-                  {t('computerUse.refreshStatus')}
-                </Button>
-              </ConfigPageRow>
             </>
           ) : null}
         </ConfigPageSection>

src/web-ui/src/infrastructure/config/components/common/ConfigPageLayout.tsx

@@ -92,6 +92,11 @@ export interface ConfigPageRowProps {
   multiline?: boolean;
   /** Flip to 3/7 ratio giving the control column more space */
   wide?: boolean;
+  /**
+   * ~40% label / ~60% control — middle ground between default (7:3) and wide (2:8).
+   * Use when the label must stay on one line (e.g. two-word titles) and controls need room.
+   */
+  balanced?: boolean;
 }
 
 export const ConfigPageRow: React.FC<ConfigPageRowProps> = ({
@@ -102,17 +107,21 @@ export const ConfigPageRow: React.FC<ConfigPageRowProps> = ({
   align = 'start',
   multiline = false,
   wide = false,
+  balanced = false,
 }) => {
   const cls = [
     'bitfun-config-page-row',
     `bitfun-config-page-row--${align}`,
     multiline && 'bitfun-config-page-row--multiline',
     wide && 'bitfun-config-page-row--wide',
+    balanced && 'bitfun-config-page-row--balanced',
     className,
   ].filter(Boolean).join(' ');
 
   const gridStyle: React.CSSProperties | undefined = wide
     ? { gridTemplateColumns: 'minmax(0, 2fr) minmax(0, 8fr)' }
+    : balanced
+    ? { gridTemplateColumns: 'minmax(0, 2fr) minmax(0, 3fr)' }
     : multiline
     ? { gridTemplateColumns: '1fr' }
     : undefined;

src/web-ui/src/locales/en-US/settings/session-config.json

@@ -14,15 +14,15 @@
   },
   "computerUse": {
     "sectionTitle": "Computer use (Claw)",
-    "sectionDescription": "Let the assistant capture the screen and control the mouse and keyboard in BitFun desktop. Requires macOS Accessibility and Screen Recording (or equivalent on other platforms). Screenshots in tool results need a primary model with Anthropic API format.",
+    "sectionDescription": "Let the assistant capture the screen and control the mouse and keyboard in BitFun desktop.",
     "enable": "Enable Computer use",
     "enableDesc": "When off, the ComputerUse tool stays disabled even in Claw mode.",
     "accessibility": "Accessibility",
     "screenCapture": "Screen recording",
     "granted": "Granted",
     "notGranted": "Not granted",
     "request": "Request",
-    "openSettings": "Open System Settings",
+    "openSettings": "Setting",
     "refreshStatus": "Refresh status",
     "desktopOnly": "Computer use settings are only available in the BitFun desktop app.",
     "platformNote": "Note"