diff --git a/.env.example b/.env.example index f65de21..bbd8a96 100644 --- a/.env.example +++ b/.env.example @@ -11,12 +11,19 @@ OPENAI_API_KEY=sk-your-openai-key # ============================================================================== # REQUIRED: OpenClaw Gateway # ============================================================================== -# The URL where your OpenClaw gateway is running -# If running on the same machine as the robot, use the host machine's IP -OPENCLAW_GATEWAY_URL=http://192.168.1.100:18789 +# The URL where your OpenClaw gateway is running. +# When ClawBody runs ON the robot, this must be reachable over the network: +# use your gateway machine's LAN IP or mDNS name (e.g. http://my-macbook.local:18789), +# NOT localhost/127.0.0.1 (that would point at the robot itself). +# The gateway must also listen on the LAN: set gateway.bind to "lan" in +# ~/.openclaw/openclaw.json (default is loopback-only) and restart the gateway. +OPENCLAW_GATEWAY_URL=http://*************:18789 # Your OpenClaw gateway authentication token # Find this in ~/.openclaw/openclaw.json under gateway.token +# Note: on first connection from the robot, the gateway will hold a pending +# device pairing request. Approve it once on the gateway machine with: +# openclaw devices list && openclaw devices approve OPENCLAW_TOKEN=your-gateway-token # Agent ID to use (default: main) @@ -33,8 +40,9 @@ OPENCLAW_SESSION_KEY=main # OpenAI Realtime voice (alloy, echo, fable, onyx, nova, shimmer, cedar) OPENAI_VOICE=cedar -# OpenAI model for Realtime API -OPENAI_MODEL=gpt-4o-realtime-preview-2024-12-17 +# OpenAI model for Realtime API (GA models only; the beta/preview realtime +# models were retired when OpenAI shut down the Realtime Beta API) +OPENAI_MODEL=gpt-realtime-1.5 # ============================================================================== # OPTIONAL: Features diff --git a/src/reachy_mini_openclaw/openai_realtime.py b/src/reachy_mini_openclaw/openai_realtime.py index d6b61e7..a6c04a8 100644 --- a/src/reachy_mini_openclaw/openai_realtime.py +++ b/src/reachy_mini_openclaw/openai_realtime.py @@ -225,25 +225,33 @@ async def _run_session(self) -> None: # Fetch OpenClaw agent context (personality, memories, user info) system_instructions = await self._build_system_instructions() - async with self.client.beta.realtime.connect(model=model) as conn: + # GA Realtime API (the beta API shape was retired by OpenAI in May 2026) + async with self.client.realtime.connect(model=model) as conn: # Configure session with OpenClaw's identity + robot body capabilities tools = self._build_tools() await conn.session.update( session={ - "modalities": ["text", "audio"], + "type": "realtime", + "output_modalities": ["audio"], "instructions": system_instructions, - "voice": get_session_voice(), - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "input_audio_transcription": { - "model": "whisper-1", - }, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 600, + "audio": { + "input": { + "format": {"type": "audio/pcm", "rate": OPENAI_SAMPLE_RATE}, + "transcription": { + "model": "gpt-4o-transcribe", + }, + "turn_detection": { + "type": "server_vad", + "threshold": 0.5, + "prefix_padding_ms": 300, + "silence_duration_ms": 600, + }, + }, + "output": { + "format": {"type": "audio/pcm", "rate": OPENAI_SAMPLE_RATE}, + "voice": get_session_voice(), + }, }, "tools": tools, "tool_choice": "auto", @@ -321,8 +329,8 @@ async def _handle_event(self, event: Any) -> None: self._speaking = True logger.debug("Response started") - # Audio output from TTS - if event_type == "response.audio.delta": + # Audio output from TTS (GA event name; was response.audio.delta in beta) + if event_type == "response.output_audio.delta": # Audio arriving means we have a response - stop thinking animation self.deps.movement_manager.set_processing(False) @@ -340,11 +348,11 @@ async def _handle_event(self, event: Any) -> None: await self.output_queue.put((OPENAI_SAMPLE_RATE, audio_data)) # Response text (for logging and UI) - if event_type == "response.audio_transcript.delta": + if event_type == "response.output_audio_transcript.delta": # Streaming transcript of what's being said pass # Could log incrementally if needed - if event_type == "response.audio_transcript.done": + if event_type == "response.output_audio_transcript.done": response_text = event.transcript logger.info("Assistant: %s", response_text[:100] if len(response_text) > 100 else response_text) self._last_assistant_response = response_text # Track for sync diff --git a/src/reachy_mini_openclaw/openclaw_bridge.py b/src/reachy_mini_openclaw/openclaw_bridge.py index af512b1..03f07ed 100644 --- a/src/reachy_mini_openclaw/openclaw_bridge.py +++ b/src/reachy_mini_openclaw/openclaw_bridge.py @@ -9,8 +9,12 @@ import json import asyncio +import base64 +import hashlib import logging +import time import uuid +from pathlib import Path from typing import Optional, Any, AsyncIterator from dataclasses import dataclass @@ -21,7 +25,97 @@ logger = logging.getLogger(__name__) # Protocol version supported by this client -PROTOCOL_VERSION = 3 +PROTOCOL_VERSION = 4 + +# Where this device's Ed25519 identity is persisted. The OpenClaw gateway +# requires remote (non-localhost) clients to present a stable device identity; +# the device must be approved (paired) once on the gateway, after which the +# requested scopes are granted to this identity. +DEVICE_IDENTITY_PATH = Path.home() / ".clawbody" / "device-identity.json" + + +def _b64url(data: bytes) -> str: + """Base64url-encode without padding (matches OpenClaw's encoding).""" + return base64.urlsafe_b64encode(data).decode("ascii").rstrip("=") + + +def _b64url_decode(data: str) -> bytes: + """Base64url-decode, tolerating missing padding.""" + return base64.urlsafe_b64decode(data + "=" * (-len(data) % 4)) + + +def _load_or_create_device_identity() -> dict: + """Load the persistent device identity, creating it on first use.""" + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + if DEVICE_IDENTITY_PATH.exists(): + data = json.loads(DEVICE_IDENTITY_PATH.read_text()) + private_key = Ed25519PrivateKey.from_private_bytes( + _b64url_decode(data["privateKey"]) + ) + else: + private_key = Ed25519PrivateKey.generate() + raw_private = private_key.private_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PrivateFormat.Raw, + encryption_algorithm=serialization.NoEncryption(), + ) + DEVICE_IDENTITY_PATH.parent.mkdir(parents=True, exist_ok=True) + DEVICE_IDENTITY_PATH.touch(mode=0o600, exist_ok=True) + DEVICE_IDENTITY_PATH.write_text( + json.dumps({"privateKey": _b64url(raw_private)}) + ) + + raw_public = private_key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + return { + "private_key": private_key, + "public_key_b64url": _b64url(raw_public), + # Device id is the sha256 hex digest of the raw public key + "device_id": hashlib.sha256(raw_public).hexdigest(), + } + + +def _build_device_auth( + nonce: str, + client_info: dict, + role: str, + scopes: list, + token: Optional[str], +) -> Optional[dict]: + """Build the signed `device` connect param (v3 signature payload).""" + try: + identity = _load_or_create_device_identity() + signed_at = int(time.time() * 1000) + payload_v3 = "|".join( + [ + "v3", + identity["device_id"], + client_info["id"], + client_info["mode"], + role, + ",".join(scopes), + str(signed_at), + token or "", + nonce, + client_info.get("platform", "").lower(), + "", # deviceFamily (unset) + ] + ) + signature = identity["private_key"].sign(payload_v3.encode("utf-8")) + return { + "id": identity["device_id"], + "publicKey": identity["public_key_b64url"], + "signature": _b64url(signature), + "signedAt": signed_at, + "nonce": nonce, + } + except Exception as e: + logger.warning("Device identity unavailable, connecting without it: %s", e) + return None @dataclass @@ -146,31 +240,49 @@ async def connect(self) -> bool: close_timeout=5, ) - # 1. Receive challenge + # 1. Receive challenge (carries the nonce for device signing) raw = await asyncio.wait_for(self._ws.recv(), timeout=10) challenge = json.loads(raw) + nonce = None if challenge.get("event") != "connect.challenge": logger.warning("Unexpected first frame: %s", challenge.get("event")) + else: + payload = challenge.get("payload") or {} + nonce = payload.get("nonce") or challenge.get("nonce") + + # 2. Send connect request (with signed device identity when possible) + client_info = { + "id": "gateway-client", + "version": "1.0.0", + "platform": "linux", + "mode": "backend", + } + role = "operator" + scopes = ["operator.read", "operator.write"] + device = ( + _build_device_auth( + nonce, client_info, role, scopes, self.gateway_token + ) + if nonce + else None + ) - # 2. Send connect request req_id = str(uuid.uuid4()) + params = { + "minProtocol": PROTOCOL_VERSION, + "maxProtocol": PROTOCOL_VERSION, + "auth": {"token": self.gateway_token} if self.gateway_token else {}, + "client": client_info, + "role": role, + "scopes": scopes, + } + if device: + params["device"] = device connect_req = { "type": "req", "id": req_id, "method": "connect", - "params": { - "minProtocol": PROTOCOL_VERSION, - "maxProtocol": PROTOCOL_VERSION, - "auth": {"token": self.gateway_token} if self.gateway_token else {}, - "client": { - "id": "openclaw-control-ui", - "version": "1.0.0", - "platform": "linux", - "mode": "webchat", - }, - "role": "operator", - "scopes": ["chat", "operator.write", "operator.read"], - }, + "params": params, } await self._ws.send(json.dumps(connect_req)) @@ -200,6 +312,15 @@ async def connect(self) -> bool: err.get("code"), err.get("message"), ) + if err.get("code") == "NOT_PAIRED" or "pairing" in str( + err.get("message", "") + ).lower(): + logger.warning( + "This robot's device identity is awaiting approval on the " + "gateway. On the gateway machine, approve the pending " + "device (e.g. `openclaw devices` / Control UI -> Devices), " + "then reconnect." + ) await self._close_ws() return False diff --git a/src/reachy_mini_openclaw/vision/mediapipe_tracker.py b/src/reachy_mini_openclaw/vision/mediapipe_tracker.py index cbecc21..7dac933 100644 --- a/src/reachy_mini_openclaw/vision/mediapipe_tracker.py +++ b/src/reachy_mini_openclaw/vision/mediapipe_tracker.py @@ -62,7 +62,7 @@ def get_head_position( try: # Convert BGR to RGB for MediaPipe - rgb_img = img[:, :, ::-1] + rgb_img = np.ascontiguousarray(img[:, :, ::-1]) # Run face detection results = self.face_detection.process(rgb_img)