Fix ClawBody on current OpenAI + OpenClaw stacks (GA Realtime API, gateway protocol v4, device identity)

.env.example

@@ -11,12 +11,19 @@ OPENAI_API_KEY=sk-your-openai-key
 # ==============================================================================
 # REQUIRED: OpenClaw Gateway
 # ==============================================================================
-# The URL where your OpenClaw gateway is running
-# If running on the same machine as the robot, use the host machine's IP
-OPENCLAW_GATEWAY_URL=http://192.168.1.100:18789
+# The URL where your OpenClaw gateway is running.
+# When ClawBody runs ON the robot, this must be reachable over the network:
+# use your gateway machine's LAN IP or mDNS name (e.g. http://my-macbook.local:18789),
+# NOT localhost/127.0.0.1 (that would point at the robot itself).
+# The gateway must also listen on the LAN: set gateway.bind to "lan" in
+# ~/.openclaw/openclaw.json (default is loopback-only) and restart the gateway.
+OPENCLAW_GATEWAY_URL=http://*************:18789
 
 # Your OpenClaw gateway authentication token
 # Find this in ~/.openclaw/openclaw.json under gateway.token
+# Note: on first connection from the robot, the gateway will hold a pending
+# device pairing request. Approve it once on the gateway machine with:
+#   openclaw devices list && openclaw devices approve <request-id>
 OPENCLAW_TOKEN=your-gateway-token
 
 # Agent ID to use (default: main)
@@ -33,8 +40,9 @@ OPENCLAW_SESSION_KEY=main
 # OpenAI Realtime voice (alloy, echo, fable, onyx, nova, shimmer, cedar)
 OPENAI_VOICE=cedar
 
-# OpenAI model for Realtime API
-OPENAI_MODEL=gpt-4o-realtime-preview-2024-12-17
+# OpenAI model for Realtime API (GA models only; the beta/preview realtime
+# models were retired when OpenAI shut down the Realtime Beta API)
+OPENAI_MODEL=gpt-realtime-1.5
 
 # ==============================================================================
 # OPTIONAL: Features

src/reachy_mini_openclaw/openai_realtime.py

@@ -225,25 +225,33 @@ async def _run_session(self) -> None:
         # Fetch OpenClaw agent context (personality, memories, user info)
         system_instructions = await self._build_system_instructions()
 
-        async with self.client.beta.realtime.connect(model=model) as conn:
+        # GA Realtime API (the beta API shape was retired by OpenAI in May 2026)
+        async with self.client.realtime.connect(model=model) as conn:
             # Configure session with OpenClaw's identity + robot body capabilities
             tools = self._build_tools()
 
             await conn.session.update(
                 session={
-                    "modalities": ["text", "audio"],
+                    "type": "realtime",
+                    "output_modalities": ["audio"],
                     "instructions": system_instructions,
-                    "voice": get_session_voice(),
-                    "input_audio_format": "pcm16",
-                    "output_audio_format": "pcm16",
-                    "input_audio_transcription": {
-                        "model": "whisper-1",
-                    },
-                    "turn_detection": {
-                        "type": "server_vad",
-                        "threshold": 0.5,
-                        "prefix_padding_ms": 300,
-                        "silence_duration_ms": 600,
+                    "audio": {
+                        "input": {
+                            "format": {"type": "audio/pcm", "rate": OPENAI_SAMPLE_RATE},
+                            "transcription": {
+                                "model": "gpt-4o-transcribe",
+                            },
+                            "turn_detection": {
+                                "type": "server_vad",
+                                "threshold": 0.5,
+                                "prefix_padding_ms": 300,
+                                "silence_duration_ms": 600,
+                            },
+                        },
+                        "output": {
+                            "format": {"type": "audio/pcm", "rate": OPENAI_SAMPLE_RATE},
+                            "voice": get_session_voice(),
+                        },
                     },
                     "tools": tools,
                     "tool_choice": "auto",
@@ -321,8 +329,8 @@ async def _handle_event(self, event: Any) -> None:
             self._speaking = True
             logger.debug("Response started")
 
-        # Audio output from TTS
-        if event_type == "response.audio.delta":
+        # Audio output from TTS (GA event name; was response.audio.delta in beta)
+        if event_type == "response.output_audio.delta":
             # Audio arriving means we have a response - stop thinking animation
             self.deps.movement_manager.set_processing(False)
 
@@ -340,11 +348,11 @@ async def _handle_event(self, event: Any) -> None:
             await self.output_queue.put((OPENAI_SAMPLE_RATE, audio_data))
 
         # Response text (for logging and UI)
-        if event_type == "response.audio_transcript.delta":
+        if event_type == "response.output_audio_transcript.delta":
             # Streaming transcript of what's being said
             pass  # Could log incrementally if needed
 
-        if event_type == "response.audio_transcript.done":
+        if event_type == "response.output_audio_transcript.done":
             response_text = event.transcript
             logger.info("Assistant: %s", response_text[:100] if len(response_text) > 100 else response_text)
             self._last_assistant_response = response_text  # Track for sync

src/reachy_mini_openclaw/openclaw_bridge.py

@@ -9,8 +9,12 @@
 
 import json
 import asyncio
+import base64
+import hashlib
 import logging
+import time
 import uuid
+from pathlib import Path
 from typing import Optional, Any, AsyncIterator
 from dataclasses import dataclass
 
@@ -21,7 +25,97 @@
 logger = logging.getLogger(__name__)
 
 # Protocol version supported by this client
-PROTOCOL_VERSION = 3
+PROTOCOL_VERSION = 4
+
+# Where this device's Ed25519 identity is persisted. The OpenClaw gateway
+# requires remote (non-localhost) clients to present a stable device identity;
+# the device must be approved (paired) once on the gateway, after which the
+# requested scopes are granted to this identity.
+DEVICE_IDENTITY_PATH = Path.home() / ".clawbody" / "device-identity.json"
+
+
+def _b64url(data: bytes) -> str:
+    """Base64url-encode without padding (matches OpenClaw's encoding)."""
+    return base64.urlsafe_b64encode(data).decode("ascii").rstrip("=")
+
+
+def _b64url_decode(data: str) -> bytes:
+    """Base64url-decode, tolerating missing padding."""
+    return base64.urlsafe_b64decode(data + "=" * (-len(data) % 4))
+
+
+def _load_or_create_device_identity() -> dict:
+    """Load the persistent device identity, creating it on first use."""
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+    if DEVICE_IDENTITY_PATH.exists():
+        data = json.loads(DEVICE_IDENTITY_PATH.read_text())
+        private_key = Ed25519PrivateKey.from_private_bytes(
+            _b64url_decode(data["privateKey"])
+        )
+    else:
+        private_key = Ed25519PrivateKey.generate()
+        raw_private = private_key.private_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PrivateFormat.Raw,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+        DEVICE_IDENTITY_PATH.parent.mkdir(parents=True, exist_ok=True)
+        DEVICE_IDENTITY_PATH.touch(mode=0o600, exist_ok=True)
+        DEVICE_IDENTITY_PATH.write_text(
+            json.dumps({"privateKey": _b64url(raw_private)})
+        )
+
+    raw_public = private_key.public_key().public_bytes(
+        encoding=serialization.Encoding.Raw,
+        format=serialization.PublicFormat.Raw,
+    )
+    return {
+        "private_key": private_key,
+        "public_key_b64url": _b64url(raw_public),
+        # Device id is the sha256 hex digest of the raw public key
+        "device_id": hashlib.sha256(raw_public).hexdigest(),
+    }
+
+
+def _build_device_auth(
+    nonce: str,
+    client_info: dict,
+    role: str,
+    scopes: list,
+    token: Optional[str],
+) -> Optional[dict]:
+    """Build the signed `device` connect param (v3 signature payload)."""
+    try:
+        identity = _load_or_create_device_identity()
+        signed_at = int(time.time() * 1000)
+        payload_v3 = "|".join(
+            [
+                "v3",
+                identity["device_id"],
+                client_info["id"],
+                client_info["mode"],
+                role,
+                ",".join(scopes),
+                str(signed_at),
+                token or "",
+                nonce,
+                client_info.get("platform", "").lower(),
+                "",  # deviceFamily (unset)
+            ]
+        )
+        signature = identity["private_key"].sign(payload_v3.encode("utf-8"))
+        return {
+            "id": identity["device_id"],
+            "publicKey": identity["public_key_b64url"],
+            "signature": _b64url(signature),
+            "signedAt": signed_at,
+            "nonce": nonce,
+        }
+    except Exception as e:
+        logger.warning("Device identity unavailable, connecting without it: %s", e)
+        return None
 
 
 @dataclass
@@ -146,31 +240,49 @@ async def connect(self) -> bool:
                 close_timeout=5,
             )
 
-            # 1. Receive challenge
+            # 1. Receive challenge (carries the nonce for device signing)
             raw = await asyncio.wait_for(self._ws.recv(), timeout=10)
             challenge = json.loads(raw)
+            nonce = None
             if challenge.get("event") != "connect.challenge":
                 logger.warning("Unexpected first frame: %s", challenge.get("event"))
+            else:
+                payload = challenge.get("payload") or {}
+                nonce = payload.get("nonce") or challenge.get("nonce")
+
+            # 2. Send connect request (with signed device identity when possible)
+            client_info = {
+                "id": "gateway-client",
+                "version": "1.0.0",
+                "platform": "linux",
+                "mode": "backend",
+            }
+            role = "operator"
+            scopes = ["operator.read", "operator.write"]
+            device = (
+                _build_device_auth(
+                    nonce, client_info, role, scopes, self.gateway_token
+                )
+                if nonce
+                else None
+            )
 
-            # 2. Send connect request
             req_id = str(uuid.uuid4())
+            params = {
+                "minProtocol": PROTOCOL_VERSION,
+                "maxProtocol": PROTOCOL_VERSION,
+                "auth": {"token": self.gateway_token} if self.gateway_token else {},
+                "client": client_info,
+                "role": role,
+                "scopes": scopes,
+            }
+            if device:
+                params["device"] = device
             connect_req = {
                 "type": "req",
                 "id": req_id,
                 "method": "connect",
-                "params": {
-                    "minProtocol": PROTOCOL_VERSION,
-                    "maxProtocol": PROTOCOL_VERSION,
-                    "auth": {"token": self.gateway_token} if self.gateway_token else {},
-                    "client": {
-                        "id": "openclaw-control-ui",
-                        "version": "1.0.0",
-                        "platform": "linux",
-                        "mode": "webchat",
-                    },
-                    "role": "operator",
-                    "scopes": ["chat", "operator.write", "operator.read"],
-                },
+                "params": params,
             }
             await self._ws.send(json.dumps(connect_req))
 
@@ -200,6 +312,15 @@ async def connect(self) -> bool:
                     err.get("code"),
                     err.get("message"),
                 )
+                if err.get("code") == "NOT_PAIRED" or "pairing" in str(
+                    err.get("message", "")
+                ).lower():
+                    logger.warning(
+                        "This robot's device identity is awaiting approval on the "
+                        "gateway. On the gateway machine, approve the pending "
+                        "device (e.g. `openclaw devices` / Control UI -> Devices), "
+                        "then reconnect."
+                    )
                 await self._close_ws()
                 return False
 

src/reachy_mini_openclaw/vision/mediapipe_tracker.py

@@ -62,7 +62,7 @@ def get_head_position(
 
         try:
             # Convert BGR to RGB for MediaPipe
-            rgb_img = img[:, :, ::-1]
+            rgb_img = np.ascontiguousarray(img[:, :, ::-1])
 
             # Run face detection
             results = self.face_detection.process(rgb_img)