diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index db2c31d0e..462d60d91 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -33,7 +33,7 @@ jobs:
- name: Install test dependencies
run: uv pip install --system -e ".[test]"
- name: Install Flywheel export dependencies
- run: uv pip install --system -e ".[flywheel]" --torch-backend cpu
+ run: uv pip install --system -e ".[flywheel]" "opencv-python-headless==4.10.0.84" --torch-backend cpu
- name: Run unit tests with coverage
run: coverage run --source=rpent,robots -m pytest tests/unit_tests -v
- name: Generate coverage report
diff --git a/calibration_tools/base_handeye_collect.py b/calibration_tools/base_handeye_collect.py
index 49ed0f9aa..8135022aa 100644
--- a/calibration_tools/base_handeye_collect.py
+++ b/calibration_tools/base_handeye_collect.py
@@ -19,6 +19,7 @@
import base64
import json
import logging
+import re
import shlex
import shutil
import subprocess
@@ -39,6 +40,7 @@
check_opencv,
check_state,
delta,
+ rigid_transform,
transform,
write_json,
)
@@ -94,12 +96,16 @@ def board_metadata(self) -> Record:
class Collector:
"""Own acquisition state without mutating other imported modules."""
- def __init__(self, config: CaptureConfig, root: Path) -> None:
+ def __init__(
+ self, config: CaptureConfig, root: Path, *, resume: bool = False
+ ) -> None:
"""Initialize an isolated session; hardware is read only on request."""
self.config = config
self.root = root
self.pose_key = f"T_{config.arm}_base_ee"
self.samples: list[Record] = []
+ self.reference: Record | None = None
+ self.next_index = 1
self.last: Record = {}
self.lock = threading.Lock()
self.board = cv2.aruco.CharucoBoard(
@@ -109,6 +115,93 @@ def __init__(self, config: CaptureConfig, root: Path) -> None:
cv2.aruco.getPredefinedDictionary(getattr(cv2.aruco, config.dictionary)),
)
self.board.setLegacyPattern(False)
+ if resume:
+ self._resume()
+ elif root.exists():
+ raise FileExistsError("Output directory already exists; use --resume")
+
+ def _resume(self) -> None:
+ """Load saved samples without changing their identities or files."""
+ if not self.root.is_dir():
+ raise ValueError("--resume requires an existing session directory")
+ reference: Record | None = None
+ indices: set[int] = set()
+ for parent in (self.root, self.root / "excluded"):
+ for folder in sorted(parent.glob("sample_*")):
+ match = re.fullmatch(r"sample_([0-9]+)", folder.name)
+ if not match or not folder.is_dir() or folder.is_symlink():
+ raise ValueError(f"Invalid sample directory: {folder}")
+ index = int(match[1])
+ if (
+ index < 1
+ or index in indices
+ or folder.name != f"sample_{index:03d}"
+ ):
+ raise ValueError(f"Invalid or duplicate sample index: {folder}")
+ indices.add(index)
+ self.next_index = max(self.next_index, index + 1)
+ record = json.loads(
+ (folder / "sample.json").read_text(encoding="utf-8")
+ )
+ if (
+ type(record["index"]) is not int
+ or record["index"] != index
+ or record["arm"] != self.config.arm
+ or record["calibration_mode"] != "eye_to_hand"
+ or record["camera"]["serial"] != self.config.camera_serial
+ or record["board"] != self.config.board_metadata
+ ):
+ raise ValueError(f"Sample configuration does not match: {folder}")
+ if reference is None:
+ reference = record
+ check_camera(record["camera"], reference["camera"])
+ for key in ("robot_before", "robot_after"):
+ check_state(record[key])
+ if not np.allclose(
+ record[key]["F_T_EE"],
+ reference["robot_before"]["F_T_EE"],
+ rtol=0,
+ atol=1e-8,
+ ):
+ raise ValueError(f"End-effector frame changed: {folder}")
+ pose = rigid_transform(record[self.pose_key], self.pose_key)
+ if not np.allclose(
+ pose, transform(record["robot_before"]), rtol=0, atol=1e-8
+ ):
+ raise ValueError(
+ f"Stored robot pose disagrees with state: {folder}"
+ )
+ rigid_transform(record["T_camera_board"], "T_camera_board")
+ for name in ("color.png", "annotated.png"):
+ if not (folder / name).is_file():
+ raise ValueError(f"Missing sample image: {folder / name}")
+ if parent == self.root:
+ self.samples.append(record)
+ self.samples.sort(key=lambda sample: sample["index"])
+ self.reference = reference
+ self.last = {"resumed": True, "count": len(self.samples)}
+
+ def delete_sample(self, index: int) -> Record:
+ """Exclude a selected sample, preserving its files for manual recovery."""
+ if type(index) is not int or index < 1:
+ raise ValueError("Sample index must be a positive integer")
+ sample = next((s for s in self.samples if s["index"] == index), None)
+ if sample is None:
+ raise ValueError(f"No active sample with index {index}")
+ source = self.root / f"sample_{index:03d}"
+ destination = self.root / "excluded" / source.name
+ if destination.exists():
+ raise FileExistsError(f"Excluded sample already exists: {destination}")
+ destination.parent.mkdir(exist_ok=True)
+ source.rename(destination)
+ self.samples.remove(sample)
+ LOGGER.info("Excluded sample %d; files retained at %s", index, destination)
+ return {
+ "deleted": True,
+ "index": index,
+ "count": len(self.samples),
+ "archived": str(destination),
+ }
def read_state(self) -> Record:
"""Run the configured readOnce probe with a bounded local timeout."""
@@ -150,9 +243,9 @@ def sample(self) -> Record:
raise ValueError("采样窗口超过 3 秒，请检查网络后重试")
if before["F_T_EE"] != after["F_T_EE"]:
raise ValueError("末端坐标定义在采样期间发生变化")
- if self.samples and not np.allclose(
+ if self.reference is not None and not np.allclose(
before["F_T_EE"],
- self.samples[0]["robot_before"]["F_T_EE"],
+ self.reference["robot_before"]["F_T_EE"],
atol=1e-8,
rtol=0,
):
@@ -161,7 +254,7 @@ def sample(self) -> Record:
dm, dr = delta(a, np.array(old[self.pose_key]))
if dm < 0.005 and dr < np.deg2rad(5):
raise ValueError("与已有姿态过于接近，请改变姿态后采集；不要重复点击")
- check_camera(camera, self.samples[0]["camera"] if self.samples else camera)
+ check_camera(camera, self.reference["camera"] if self.reference else camera)
png = base64.b64decode(camera.pop("png_b64"), validate=True)
im = cv2.imdecode(np.frombuffer(png, np.uint8), cv2.IMREAD_COLOR)
if im is None or im.shape[:2] != (camera["height"], camera["width"]):
@@ -203,7 +296,7 @@ def sample(self) -> Record:
t[:3, :3] = rot
t[:3, 3] = tv.ravel()
record = {
- "index": len(self.samples) + 1,
+ "index": self.next_index,
"host_start_s": start,
"host_end_s": time.time(),
"robot_before": before,
@@ -241,6 +334,9 @@ def sample(self) -> Record:
if temporary.exists():
shutil.rmtree(temporary)
self.samples.append(record)
+ if self.reference is None:
+ self.reference = record
+ self.next_index += 1
return {
"accepted": True,
"count": len(self.samples),
@@ -255,6 +351,14 @@ def status(self) -> Record:
"count": len(self.samples),
"session": str(self.root),
"last": self.last,
+ "samples": [
+ {
+ "index": sample["index"],
+ "corners": sample["corners"],
+ "reprojection_rms_px": round(sample["reprojection_rms_px"], 3),
+ }
+ for sample in self.samples
+ ],
}
def preview(self) -> bytes:
@@ -284,19 +388,47 @@ def preview(self) -> bytes:
PAGE = b"""Hand-eye calibration
-
+
Stationary hand-eye calibration
Move the arm manually, release guidance, and wait two seconds.
This collector only reads robot state. Keep the board visible and collect varied rotations.
+
Saved samples
+
Preview a sample before excluding it. Excluded samples are kept in the session's
+excluded/ directory and are not used by the solver. Re-solve after changing samples.
+
Sample
Corners
RMS (px)
Actions
A recorded sample is not a successful calibration. Solve and validate on new poses.
"""
@@ -330,6 +462,20 @@ def do_GET(self) -> None:
with collector.lock:
status = collector.status()
self.reply(200, json.dumps(status, allow_nan=False).encode())
+ elif match := re.fullmatch(r"/samples/([0-9]+)/annotated\.png", self.path):
+ index = int(match[1])
+ with collector.lock:
+ if not any(s["index"] == index for s in collector.samples):
+ self.send_error(404)
+ return
+ try:
+ data = (
+ collector.root / f"sample_{index:03d}" / "annotated.png"
+ ).read_bytes()
+ except OSError as error:
+ self.reply(503, json.dumps({"error": str(error)}).encode())
+ return
+ self.reply(200, data, "image/png")
elif self.path.startswith("/frame.jpg"):
try:
self.reply(200, collector.preview(), "image/jpeg")
@@ -339,16 +485,21 @@ def do_GET(self) -> None:
self.send_error(404)
def do_POST(self) -> None:
- """Record one sample while rejecting concurrent acquisition."""
- if self.path != "/sample":
+ """Serialize sample capture and recoverable exclusion."""
+ deletion = re.fullmatch(r"/samples/([0-9]+)/delete", self.path)
+ if self.path != "/sample" and deletion is None:
self.send_error(404)
return
if not collector.lock.acquire(blocking=False):
- self.reply(409, b'{"error":"sample in progress"}')
+ self.reply(409, b'{"error":"sample operation in progress"}')
return
try:
try:
- collector.last = collector.sample()
+ collector.last = (
+ collector.delete_sample(int(deletion[1]))
+ if deletion
+ else collector.sample()
+ )
code = 200
except (
ValueError,
@@ -357,7 +508,7 @@ def do_POST(self) -> None:
cv2.error,
subprocess.SubprocessError,
) as error:
- LOGGER.warning("Sample rejected: %s", error)
+ LOGGER.warning("Sample operation rejected: %s", error)
collector.last = {"accepted": False, "error": str(error)}
code = 422
self.reply(code, json.dumps(collector.last, allow_nan=False).encode())
@@ -386,6 +537,9 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
help="Optional libfranka shared-library directory on the reader host",
)
parser.add_argument("--output", type=Path)
+ parser.add_argument(
+ "--resume", action="store_true", help="Resume --output with unchanged setup"
+ )
parser.add_argument("--port", type=int, default=8767)
parser.add_argument("--squares-x", type=int, default=6)
parser.add_argument("--squares-y", type=int, default=8)
@@ -441,14 +595,18 @@ def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
add_arguments(parser)
args = parser.parse_args()
+ if args.resume and args.output is None:
+ parser.error("--resume requires --output")
config = build_config(args)
root = args.output or Path(__file__).resolve().parent / "sessions" / (
datetime.now().strftime("%Y%m%d_%H%M%S_%f") + f"_{config.arm}_eye_to_hand"
)
- if root.exists():
- parser.error("Output directory already exists; use a new session directory")
+ try:
+ collector = Collector(config, root, resume=args.resume)
+ except (ValueError, OSError, KeyError) as error:
+ parser.error(str(error))
logging.basicConfig(level=logging.INFO)
- serve(Collector(config, root), args.port)
+ serve(collector, args.port)
if __name__ == "__main__":
diff --git a/robots/dual_franka/config/example.yaml b/robots/dual_franka/config/example.yaml
index 4e4f4b6e8..e9de4d86f 100644
--- a/robots/dual_franka/config/example.yaml
+++ b/robots/dual_franka/config/example.yaml
@@ -1,6 +1,7 @@
# RPent example config for a dual-Franka setup.
robot:
realtime_config: ignore
+ # Omit compliance to use RLinf defaults; tune overrides in your deployment YAML.
arms:
left:
ip: "172.16.0.2"
@@ -14,6 +15,8 @@ robot:
connection: "/dev/serial/by-id/usb-FTDI_USB_TO_RS-485_DAAIT8PU-if00-port0"
cameras:
+ # Wrist cameras connect to their arm's controller node; the base and
+ # perception cameras connect to the environment node.
main: left_wrist_0_rgb
# Agent-facing view policy. ``inline`` views are sent directly in planner
# tool results; ``auxiliary`` views are saved as artifacts and can be inspected
@@ -52,7 +55,8 @@ workspace:
- [0.8, 0.8, 0.7, 3.1416, 3.1416, 3.1416]
joint_health:
- # Tune these guard rails for the mounting, reset posture, tool, and task.
+ # Advisory heuristics, not hardware limits or an automatic reset trigger.
+ # Tune for the mounting, reset posture, tool, and task.
thresholds:
left:
warning_drift_l2: 2.4
diff --git a/robots/dual_franka/dual_franka_manual_call.py b/robots/dual_franka/dual_franka_manual_call.py
index 2a4333f61..d6ee93274 100755
--- a/robots/dual_franka/dual_franka_manual_call.py
+++ b/robots/dual_franka/dual_franka_manual_call.py
@@ -67,7 +67,11 @@
def _tool_spec_map(*, include_manual: bool = False) -> dict[str, dict[str, Any]]:
specs = {
- tool.name: {"name": tool.name, "input_schema": tool.input_schema}
+ tool.name: {
+ "name": tool.name,
+ "description": tool.description,
+ "input_schema": tool.input_schema,
+ }
for tool in DualFrankaToolkit.declared_tools()
}
if include_manual:
diff --git a/robots/dual_franka/env_server.py b/robots/dual_franka/env_server.py
index 19ba50a21..e711e8008 100644
--- a/robots/dual_franka/env_server.py
+++ b/robots/dual_franka/env_server.py
@@ -17,7 +17,6 @@
from __future__ import annotations
import queue
-import sys
import threading
import time
from typing import Any
@@ -26,15 +25,8 @@
from robots.dual_franka.runtime_config import load_runtime_config
from robots.franka.env_server import FrankaEnvFacade, main
-from rpent.utils.config import get_repo_root, get_rlinf_repo_path
from rpent.utils.serialization import to_numpy_tree
-# Resolve the RLinf checkout before the deferred ``import rlinf`` executes.
-RPENT_ROOT = get_repo_root()
-RLINF_REPO_PATH = get_rlinf_repo_path() or (RPENT_ROOT.parent / "rlinf").resolve()
-if str(RLINF_REPO_PATH) not in sys.path:
- sys.path.insert(0, str(RLINF_REPO_PATH))
-
_ARM_INDEX = {"left": 0, "right": 1}
FRANKA_JOINT_LIMIT_LOW = np.asarray(
@@ -117,6 +109,9 @@ class DualFrankaEnvWorker(Worker):
def __init__(self, cfg: Any, controller_config: dict[str, Any]):
super().__init__()
+ from robots.dual_franka.rpent_env import register_rpent_dual_franka_env
+
+ register_rpent_dual_franka_env()
self.cfg = cfg
self.controller = dict(controller_config)
self._dual_franka_calibration_bundle: dict[str, Any] | None = None
@@ -148,9 +143,15 @@ def __init__(self, cfg: Any, controller_config: dict[str, Any]):
f"per_arm_dim={self.per_arm_dim}"
)
env_config = self.env.env.call("get_wrapper_attr", "config")[0]
+ gripper_threshold = float(env_config.binary_gripper_threshold)
+ if not np.isfinite(gripper_threshold) or gripper_threshold <= 0:
+ self.env.close()
+ raise ValueError(
+ "binary_gripper_threshold must be positive so zero holds "
+ "the gripper without issuing an open/close command"
+ )
self.action_scale = np.asarray(env_config.action_scale, dtype=np.float32)
self._perception_cameras: dict[str, Any] = {}
- self._perception_camera_last_frames: dict[str, np.ndarray] = {}
self._perception_camera_meta: dict[str, dict[str, Any]] = {}
try:
self._open_perception_cameras()
@@ -255,12 +256,26 @@ def _raw_rlinf_env(self) -> Any:
return self._raw_env.unwrapped
def _refresh_robot_state(self) -> None:
- # Direct controller calls bypass the vector-env observation cache.
+ # The official accessor refreshes the arm states without moving them.
raw = self._raw_rlinf_env()
if raw.config.is_dummy:
return
- raw._left_state = raw._left_ctrl.get_state().wait()[0]
- raw._right_state = raw._right_ctrl.get_state().wait()[0]
+ raw.get_tcp_pose()
+
+ def _gripper_states(self) -> dict[str, dict[str, bool | float | None]]:
+ """Read each independent hand; arm snapshots do not contain its state."""
+ raw = self._raw_rlinf_env()
+ states: dict[str, dict[str, bool | float | None]] = {}
+ for arm in ("left", "right"):
+ if raw.config.is_dummy:
+ states[arm] = {"gripper_open": None, "gripper_position": None}
+ continue
+ hand = raw.robot.child(arm).child("end_effector")
+ states[arm] = {
+ "gripper_open": bool(hand.is_open),
+ "gripper_position": float(hand.position),
+ }
+ return states
def _reset_both_joints_no_gripper(self, reset_qpos: Any) -> dict[str, Any]:
# Reset joints directly so the episode reset does not change grippers.
@@ -270,19 +285,23 @@ def _reset_both_joints_no_gripper(self, reset_qpos: Any) -> dict[str, Any]:
def run(arm: str, ctrl: Any, qpos: Any) -> None:
try:
- results[arm] = ctrl.reset_joint(qpos).wait()
+ results[arm] = ctrl.reset_joint(qpos)
except BaseException as exc:
errors[arm] = exc
threads = [
threading.Thread(
target=run,
- args=("left", raw._left_ctrl, reset_qpos[0]),
+ args=("left", raw.robot.child("left").child("arm"), reset_qpos[0]),
daemon=True,
),
threading.Thread(
target=run,
- args=("right", raw._right_ctrl, reset_qpos[1]),
+ args=(
+ "right",
+ raw.robot.child("right").child("arm"),
+ reset_qpos[1],
+ ),
daemon=True,
),
]
@@ -336,33 +355,15 @@ def _world_arm_poses(self) -> tuple[np.ndarray, np.ndarray]:
def _arm_rot6d(self, pose: np.ndarray) -> np.ndarray:
return _matrix_to_rot6d(Rotation.from_quat(pose[3:]).as_matrix())
- @staticmethod
- def _gripper_command_from_open(gripper_open: bool | None) -> float:
- if gripper_open is None:
- return 0.0
- return 1.0 if bool(gripper_open) else -1.0
-
- def _current_gripper_commands(self) -> tuple[float, float]:
- left_state, right_state = self._arm_states()
- return (
- self._gripper_command_from_open(left_state.gripper_open),
- self._gripper_command_from_open(right_state.gripper_open),
- )
-
def _hold_action(
self,
left: np.ndarray,
right: np.ndarray,
*,
- left_grip: float | None = None,
- right_grip: float | None = None,
+ left_grip: float = 0.0,
+ right_grip: float = 0.0,
) -> np.ndarray:
- if left_grip is None or right_grip is None:
- current_left_grip, current_right_grip = self._current_gripper_commands()
- if left_grip is None:
- left_grip = current_left_grip
- if right_grip is None:
- right_grip = current_right_grip
+ """Hold both poses without reissuing a gripper command by default."""
return _pack_dual_action(
left[:3],
self._arm_rot6d(left),
@@ -478,18 +479,18 @@ def check(name: str, value: float | None, warning_op, critical_op) -> None:
@staticmethod
def _native_joint_health(state: dict[str, Any]) -> dict[str, Any]:
- def active_flags(value: Any) -> list[str]:
+ def active_flags(value: Any) -> list[str] | None:
if not isinstance(value, dict):
- return []
+ return None
return [
str(name)
for name, enabled in value.items()
if isinstance(enabled, (bool, np.bool_)) and bool(enabled)
]
- def any_array(value: Any) -> bool:
+ def any_array(value: Any) -> bool | None:
if not isinstance(value, (list, tuple, np.ndarray)):
- return False
+ return None
return bool(np.asarray(value, dtype=bool).any())
warning_reasons: list[str] = []
@@ -499,10 +500,10 @@ def any_array(value: Any) -> bool:
mode_lower = mode_text.lower() if mode_text else ""
if bool(state.get("has_errors")):
critical_reasons.append("franka_has_errors")
- for name in active_flags(state.get("current_errors")):
+ for name in active_flags(state.get("current_errors")) or []:
critical_reasons.append(f"current_error:{name}")
# Previous motion errors are warnings; active errors are critical.
- for name in active_flags(state.get("last_motion_errors")):
+ for name in active_flags(state.get("last_motion_errors")) or []:
warning_reasons.append(f"last_motion_error:{name}")
if mode_lower and any(token in mode_lower for token in ("error", "reflex")):
critical_reasons.append(f"robot_mode:{mode_text}")
@@ -516,18 +517,33 @@ def any_array(value: Any) -> bool:
warning_reasons.append("joint_contact")
if any_array(state.get("cartesian_contact")):
warning_reasons.append("cartesian_contact")
+ summary = {
+ "robot_mode": mode_text,
+ "has_errors": state.get("has_errors"),
+ "current_errors": active_flags(state.get("current_errors")),
+ "last_motion_errors": active_flags(state.get("last_motion_errors")),
+ "joint_contact": any_array(state.get("joint_contact")),
+ "cartesian_contact": any_array(state.get("cartesian_contact")),
+ "joint_collision": any_array(state.get("joint_collision")),
+ "cartesian_collision": any_array(state.get("cartesian_collision")),
+ }
+ available = any(value is not None for value in summary.values())
+ complete = all(value is not None for value in summary.values())
return {
"critical_reasons": critical_reasons,
"warning_reasons": warning_reasons,
"summary": {
- "robot_mode": mode_text,
- "has_errors": state.get("has_errors"),
- "current_errors": active_flags(state.get("current_errors")),
- "last_motion_errors": active_flags(state.get("last_motion_errors")),
- "joint_contact": any_array(state.get("joint_contact")),
- "cartesian_contact": any_array(state.get("cartesian_contact")),
- "joint_collision": any_array(state.get("joint_collision")),
- "cartesian_collision": any_array(state.get("cartesian_collision")),
+ **summary,
+ "available": available,
+ "status": (
+ "critical"
+ if critical_reasons
+ else "warning"
+ if warning_reasons
+ else "ok"
+ if complete
+ else "unknown"
+ ),
},
}
@@ -536,8 +552,9 @@ def get_robot_state(self) -> dict[str, Any]:
left, right = self._arm_states()
left_raw = to_numpy_tree(left)
right_raw = to_numpy_tree(right)
- left_out = dict(left_raw)
- right_out = dict(right_raw)
+ grippers = self._gripper_states()
+ left_out = {**left_raw, **grippers["left"]}
+ right_out = {**right_raw, **grippers["right"]}
if "tcp_pose" in left_raw:
left_out["raw_tcp_pose"] = left_raw["tcp_pose"]
left_out["raw_tcp_pose_frame"] = "left_base"
@@ -549,11 +566,11 @@ def get_robot_state(self) -> dict[str, Any]:
right_out["tcp_pose_frame"] = "right_base"
return {
"coordinate_frame": "right_base",
- "left_arm": left_out,
- "right_arm": right_out,
"joint_health": self._joint_health_from_raw(
{"left": left_raw, "right": right_raw}
),
+ "left_arm": left_out,
+ "right_arm": right_out,
"action_dim": self.action_dim,
"per_arm_dim": self.per_arm_dim,
"action_scale": self.action_scale.tolist(),
@@ -619,14 +636,11 @@ def _open_perception_cameras(self) -> None:
camera = Camera.of(info)
camera.connect()
try:
- first_frame = camera.get_frame(timeout=8)
+ camera.get_frame(timeout=8)
except Exception:
camera.disconnect()
raise
self._perception_cameras[str(alias)] = camera
- self._perception_camera_last_frames[str(alias)] = np.asarray(
- first_frame
- )
def _capture_perception_camera_snapshot(self) -> dict[str, dict[str, Any]]:
# Deferred: the helper imports RLinf camera types with the single-arm env.
@@ -639,22 +653,20 @@ def _capture_perception_camera_snapshot(self) -> dict[str, dict[str, Any]]:
}
for alias, camera in self._perception_cameras.items():
try:
- frame = camera.get_frame(timeout=2)
- self._perception_camera_last_frames[alias] = np.asarray(frame)
- except queue.Empty:
- frame = self._perception_camera_last_frames.get(alias)
- if frame is None:
- continue
- frame = np.asarray(frame)
+ reading = camera.get_observation(timeout=2)
+ except queue.Empty as exc:
+ raise RuntimeError(
+ f"Camera {alias} did not return a fresh frame; refusing cached RGBD"
+ ) from exc
+ frame = np.asarray(reading["frame"])
if frame.ndim != 3 or frame.shape[-1] < 3:
continue
rgb = frame[..., :3][..., ::-1].astype(np.uint8, copy=True)
raw_key = f"{alias}_rgb"
output["raw_frames"][raw_key] = rgb
depth = None
- if frame.shape[-1] >= 4:
- depth_scale = float(camera.depth_scale)
- depth = frame[..., 3].astype(np.float32) * depth_scale
+ if "depth" in reading:
+ depth = np.asarray(reading["depth"], dtype=np.float32).copy()
output["raw_depths"][raw_key] = depth
intrinsics = realsense_color_intrinsics(camera)
info = camera.camera_info
@@ -720,7 +732,11 @@ def move_delta(self, arm: str, delta_xyz: Any) -> dict[str, Any]:
final_world = self._pose_to_world(arm_name, final_local)
error = float(np.linalg.norm(target_world[:3] - final_world[:3]))
return {
- "ok": error <= self.controller["move_tolerance_m"],
+ # Completion describes execution, not target accuracy.
+ "ok": True,
+ "status": "completed",
+ "message": "Action completed.",
+ "target_reached": error <= self.controller["move_tolerance_m"],
"arm": arm_name,
"coordinate_frame": "right_base",
"requested_delta_xyz": requested.tolist(),
@@ -755,6 +771,26 @@ def rotate_delta(self, arm: str, delta_rpy: Any) -> dict[str, Any]:
)
iterations = 0
error = float("inf")
+ position_tolerance = self.controller["rotate_position_tolerance_m"]
+ drift_limit = self.controller["rotate_max_drift_m"]
+ max_position_error = 0.0
+ exit_reason = "iteration_limit"
+ integral = np.zeros(3, dtype=np.float64)
+ position_integral = np.zeros(3, dtype=np.float64)
+ position_integral_gain = self.controller[
+ "rotate_position_integral_gain_per_s"
+ ]
+ position_integral_limit = self.controller[
+ "rotate_position_integral_limit_m"
+ ]
+ integral_gain = self.controller["rotate_integral_gain_per_s"]
+ integral_limit = self.controller["rotate_integral_limit_rad"]
+ commanded_rot = start_rot
+ nominal_rot = start_rot
+ previous_time = time.monotonic()
+ trace = []
+ settle_s = self.controller["rotate_settle_s"]
+ reached_since = None
while iterations < max_iterations and time.time() < deadline:
left, right = self._arm_poses()
current_local = left if arm_idx == 0 else right
@@ -762,16 +798,85 @@ def rotate_delta(self, arm: str, delta_rpy: Any) -> dict[str, Any]:
current_rot = Rotation.from_quat(current_world[3:])
error_rotvec = (target_rot * current_rot.inv()).as_rotvec()
error = float(np.linalg.norm(error_rotvec))
- if error <= self.controller["rotate_tolerance_rad"]:
+ position_error = float(
+ np.linalg.norm(current_world[:3] - start_world[:3])
+ )
+ max_position_error = max(max_position_error, position_error)
+ now = time.monotonic()
+ dt = min(now - previous_time, 0.2)
+ previous_time = now
+ trace.append(
+ {
+ "iteration": iterations,
+ "error_rotvec": error_rotvec.tolist(),
+ "position_error_m": position_error,
+ "integral_rotvec": integral.tolist(),
+ "position_integral": position_integral.tolist(),
+ }
+ )
+ if position_error > drift_limit:
+ exit_reason = "position_drift"
break
- if error > max_step:
- error_rotvec = error_rotvec * (max_step / error)
- step_rot = Rotation.from_rotvec(error_rotvec) * current_rot
+ if (
+ error <= self.controller["rotate_tolerance_rad"]
+ and position_error <= position_tolerance
+ ):
+ if reached_since is None:
+ reached_since = now
+ if now - reached_since >= settle_s:
+ exit_reason = "reached"
+ break
+ else:
+ reached_since = None
+ # Finish ramping the nominal reference before compensating
+ # tracking error. The interpolation step is not an error gate.
+ nominal_delta = (target_rot * nominal_rot.inv()).as_rotvec()
+ nominal_angle = float(np.linalg.norm(nominal_delta))
+ if nominal_angle <= max_step:
+ nominal_rot = target_rot
+ else:
+ nominal_rot = (
+ Rotation.from_rotvec(nominal_delta * (max_step / nominal_angle))
+ * nominal_rot
+ )
+ if (
+ nominal_angle <= 1e-8
+ and error > self.controller["rotate_tolerance_rad"]
+ ):
+ integral += integral_gain * dt * error_rotvec
+ integral_norm = float(np.linalg.norm(integral))
+ if integral_norm > integral_limit:
+ integral *= integral_limit / integral_norm
+ reference_rot = Rotation.from_rotvec(integral) * nominal_rot
+ if nominal_angle <= 1e-8 and position_error > position_tolerance:
+ position_integral += (
+ position_integral_gain
+ * dt
+ * (start_world[:3] - current_world[:3])
+ )
+ position_integral_norm = float(np.linalg.norm(position_integral))
+ if position_integral_norm > position_integral_limit:
+ position_integral *= (
+ position_integral_limit / position_integral_norm
+ )
+ # Advance the reference independently of measured tracking lag;
+ # repeatedly stepping from the measured pose can stall before
+ # ever commanding the actual goal under impedance control.
+ command_rotvec = (reference_rot * commanded_rot.inv()).as_rotvec()
+ command_angle = float(np.linalg.norm(command_rotvec))
+ if command_angle > max_step:
+ command_rotvec *= max_step / command_angle
+ step_rot = Rotation.from_rotvec(command_rotvec) * commanded_rot
+ commanded_rot = step_rot
next_world = current_world.copy()
+ # Keep a fixed translation reference; measured drift must not
+ # become the next position target during an in-place rotation.
+ next_world[:3] = start_world[:3] + position_integral
next_world[3:] = step_rot.as_quat()
next_local = self._pose_from_world(arm_name, next_world)
action = self._hold_action(left, right)
base = arm_idx * self.per_arm_dim
+ action[base : base + 3] = next_local[:3]
action[base + 3 : base + 9] = _matrix_to_rot6d(
Rotation.from_quat(next_local[3:]).as_matrix()
)
@@ -783,8 +888,35 @@ def rotate_delta(self, arm: str, delta_rpy: Any) -> dict[str, Any]:
error = float(
(target_rot * Rotation.from_quat(final_world[3:]).inv()).magnitude()
)
+ position_error = float(np.linalg.norm(final_world[:3] - start_world[:3]))
+ max_position_error = max(max_position_error, position_error)
+ rotation_ok = error <= self.controller["rotate_tolerance_rad"]
+ position_ok = position_error <= position_tolerance
+ if max_position_error > drift_limit:
+ exit_reason = "position_drift"
+ elif (
+ rotation_ok
+ and position_ok
+ and (
+ settle_s == 0
+ or (
+ reached_since is not None
+ and time.monotonic() - reached_since >= settle_s
+ )
+ )
+ ):
+ exit_reason = "reached"
+ elif time.time() >= deadline:
+ exit_reason = "timeout"
return {
- "ok": error <= self.controller["rotate_tolerance_rad"],
+ "ok": exit_reason != "position_drift",
+ "status": "aborted" if exit_reason == "position_drift" else "completed",
+ "message": (
+ "Action aborted: position drift exceeded the protective limit."
+ if exit_reason == "position_drift"
+ else "Action completed."
+ ),
+ "target_reached": exit_reason == "reached",
"arm": arm_name,
"coordinate_frame": "right_base",
"requested_delta_rpy": requested.tolist(),
@@ -796,11 +928,23 @@ def rotate_delta(self, arm: str, delta_rpy: Any) -> dict[str, Any]:
"start_raw_tcp_pose": start_local.tolist(),
"final_raw_tcp_pose": final_local.tolist(),
"final_error_rad": error,
+ "final_position_error_m": position_error,
+ "max_position_error_m": max_position_error,
+ "position_tolerance_m": position_tolerance,
+ "position_drift_limit_m": drift_limit,
+ "settle_s": settle_s,
+ "integral_rotvec": integral.tolist(),
+ "position_integral": position_integral.tolist(),
+ "control_trace": trace,
+ "rotation_reached": rotation_ok,
+ "position_reached": position_ok,
+ "exit_reason": exit_reason,
"steps_used": iterations,
}
def set_gripper(self, arm: str, *, open: bool) -> dict[str, Any]:
arm_idx = self._arm_index(arm)
+ arm_name = ["left", "right"][arm_idx]
self._refresh_robot_state()
deadline = time.time() + self.controller["gripper_timeout_s"]
command = 1.0 if open else -1.0
@@ -815,15 +959,14 @@ def set_gripper(self, arm: str, *, open: bool) -> dict[str, Any]:
action[arm_idx * self.per_arm_dim + self.gripper_idx] = command
self.env.step(action[None, :], auto_reset=False)
time.sleep(self.controller["gripper_settle_s"])
- left_state, right_state = self._arm_states()
- state = left_state if arm_idx == 0 else right_state
- reached = bool(state.gripper_open) == bool(open)
+ state = self._gripper_states()[arm_name]
+ reached = state["gripper_open"] == bool(open)
iterations += 1
if reached:
break
return {
"ok": reached,
- "arm": ["left", "right"][arm_idx],
+ "arm": arm_name,
"target_gripper_open": bool(open),
"steps_used": iterations,
"robot_state": self.get_robot_state(),
@@ -907,7 +1050,10 @@ def command_grippers_directly(stage: str) -> dict[str, Any]:
"reason": "dummy environment",
}
- ctrls = {"left": raw._left_ctrl, "right": raw._right_ctrl}
+ hands = {
+ arm: raw.robot.child(arm).child("end_effector")
+ for arm in ("left", "right")
+ }
results: dict[str, Any] = {}
errors: dict[str, BaseException] = {}
@@ -921,11 +1067,9 @@ def run(arm: str) -> None:
}
return
try:
- ctrl = ctrls[arm]
- method = (
- ctrl.open_gripper if target_open else ctrl.close_gripper
- )
- results[arm] = method().wait()
+ hand = hands[arm]
+ method = hand.open if target_open else hand.close
+ results[arm] = method()
except BaseException as exc:
errors[arm] = exc
@@ -1047,19 +1191,10 @@ def run(arm: str) -> None:
if isinstance(value, dict) and not str(key).endswith("_rotate")
)
rotation_return_ok = all(
- bool(value.get("ok"))
+ bool(value.get("target_reached", value.get("ok")))
for key, value in return_results.items()
if isinstance(value, dict) and str(key).endswith("_rotate")
)
- final_joint_health = final_state.get("joint_health")
- final_joint_health_ok = (
- True
- if not isinstance(final_joint_health, dict)
- else all(
- (final_joint_health.get(arm) or {}).get("status") == "ok"
- for arm in ("left", "right")
- )
- )
return_ok = (not return_to_start) or (
translation_return_ok and non_rotation_return_ok
)
@@ -1077,8 +1212,7 @@ def run(arm: str) -> None:
"ok": ((not return_to_start) or return_ok)
and direct_gripper_command_ok
and gripper_restore_ok
- and gripper_preserved
- and final_joint_health_ok,
+ and gripper_preserved,
"primitive": "recover_joint_posture",
"reason": str(reason or ""),
"return_to_start": bool(return_to_start),
@@ -1097,10 +1231,8 @@ def run(arm: str) -> None:
"left_raw_tcp_pose": start_raw["left"].tolist(),
"right_raw_tcp_pose": start_raw["right"].tolist(),
"gripper_open": start_gripper_open,
- "joint_health": before_state.get("joint_health"),
},
"after_joint_reset": {
- "joint_health": after_reset_state.get("joint_health"),
"gripper_open": gripper_open_from_state(after_reset_state),
"reset_results": to_numpy_tree(reset_results),
},
@@ -1115,13 +1247,11 @@ def run(arm: str) -> None:
"non_rotation_return_ok": non_rotation_return_ok,
"rotation_return_ok": rotation_return_ok,
"rotation_is_diagnostic_only": True,
- "final_joint_health_ok": final_joint_health_ok,
"move_tolerance_m": self.controller["move_tolerance_m"],
"rotate_tolerance_rad": self.controller["rotate_tolerance_rad"],
},
"final": {
"coordinate_frame": "right_base",
- "joint_health": final_state.get("joint_health"),
"gripper_open": final_gripper_open,
"gripper_preserved": gripper_preserved,
"left_raw_tcp_pose": final_raw["left"].tolist(),
diff --git a/robots/dual_franka/perception.py b/robots/dual_franka/perception.py
index ef6f2d210..096c04c8e 100644
--- a/robots/dual_franka/perception.py
+++ b/robots/dual_franka/perception.py
@@ -831,12 +831,24 @@ def _mask_to_camera_world(
result.update({"point_xyz": None, "world_error": "empty mask"})
return result
+ # Keep the rejected mask visible too; a missing valid 3-D point must not
+ # hide where segmentation actually landed in the image.
+ result["centroid_pixel"] = [int(np.median(rows)), int(np.median(cols))]
+ result["mask_bbox_rc"] = [
+ int(rows.min()),
+ int(cols.min()),
+ int(rows.max()),
+ int(cols.max()),
+ ]
+
depths = depth[rows, cols].astype(np.float64)
valid_depth = np.isfinite(depths) & (depths > 0.0)
rows = rows[valid_depth]
cols = cols[valid_depth]
depths = depths[valid_depth]
result["valid_depth_pixels"] = int(depths.size)
+ if depths.size:
+ result["raw_depth_median_m"] = round(float(np.median(depths)), 5)
if depths.size < min_valid:
result.update(
{
diff --git a/robots/dual_franka/prompt_bundle.py b/robots/dual_franka/prompt_bundle.py
index 6ace4ffbc..2e7b3e3c2 100644
--- a/robots/dual_franka/prompt_bundle.py
+++ b/robots/dual_franka/prompt_bundle.py
@@ -33,7 +33,6 @@ def system_prompt(variables: Mapping[str, object] | None = None) -> PromptNode:
"RUNTIME": system_parts.RUNTIME,
"SAFETY RULES": Numbered(system_parts.RULES),
"CAMERA AND PROJECTION RULES": Numbered(system_parts.CAMERA_AND_PROJECTION),
- "VLA SEGMENT GATES": Numbered(system_parts.VLA_GATES),
"WORKFLOW": Numbered(system_parts.WORKFLOW),
}
return node
diff --git a/robots/dual_franka/prompts/explore.py b/robots/dual_franka/prompts/explore.py
index 4966fdbf8..6bb1aca38 100644
--- a/robots/dual_franka/prompts/explore.py
+++ b/robots/dual_franka/prompts/explore.py
@@ -22,13 +22,20 @@
from rpent.prompt.utils import Numbered
MODE = """MULTI-ATTEMPT REAL-ROBOT EXPLORATION. You are session {{session_number}}
-of {{session_max}} on {{recipe_tag}}. Complete the task through bounded physical
+of {{session_max}} on {{recipe_tag}}. Complete the task through physical
actions, learn from failed attempts, and leave evidence-backed memory.
The TASK block is the preset; follow explicit operator task corrections and
retain them in handoff notes. Task solvability is not guaranteed. Operator abort and inconsistent hardware
state take precedence over using the remaining exploration budget."""
RULES = (
+ "Acknowledge explicit operator feedback and explain the resulting plan change. "
+ "An operator may request practicing a later task stage while preserving earlier "
+ "progress. Request confirmation of that partial starting scene, not full restoration. "
+ "Do not repeat completed stages merely to satisfy the preset order. Still obtain "
+ "fresh reset confirmation before motion, inspect the scene and re-localize. "
+ "Record partial-start practice as such, not as an end-to-end successful run. "
+ "If feedback is ambiguous, ask a specific question instead of ignoring it.",
"Before the first motion in every session, request_scene_reset(reason, "
"expected_scene_state). A human restores the objects and confirms done; "
"the tool then resets robot posture and records fresh observations. A new "
@@ -36,17 +43,20 @@
"After reset, re-read the new state and re-localize. Never reuse a pixel, "
"point or TCP target from an earlier attempt. Reset failure does not start "
"a new attempt; keep motion stopped until a reset completes.",
- "Read describe_dual_franka_setup for registered tools and cameras. Use move_delta, "
- "rotate_delta, gripper tools, recover_joint_posture, back_project, optional "
- "segment, and the named vla_right_grasp/vla_handoff/vla_left_place skills. "
- "Do not assume LIBERO primitives or the older vla_grasp tool exist.",
+ "Read describe_dual_franka_setup for registered tools and cameras. Use only "
+ "the tools available for the current task; do not assume a fixed skill set.",
"Ask request_operator_verdict after apparent success or failure. The human "
"answers success, failure, continue or abort. Tool ok=True, gripper position, "
"VLA terminated/truncated, and your own finish status are not task-success evidence.",
"Any subsequent physical action invalidates the verdict. continue clears "
- "the previous verdict. Obtain another judgment before finish.",
+ "the previous verdict. A returned continue is already authorization to "
+ "resume this attempt, not a request to wait for another continue. Read its "
+ "notes and fresh observation and continue without resetting. Do not ask "
+ "again about the same answered question unless new evidence introduces a "
+ "concrete blocker. Obtain another judgment before finish.",
"For a failed attempt, archive evidence and update wip before requesting "
- "a new scene reset. Change a named lever: order, staging, target or VLA prompt/chunk budget. "
+ "a new scene reset. Change one meaningful lever, such as perception inputs "
+ "or use of available tools, within the current task requirements. "
"Use in-place recovery only when safe. Never force a restart after operator abort.",
)
@@ -119,7 +129,6 @@ def system_prompt():
"RUNTIME": base.RUNTIME,
"SAFETY RULES": Numbered(base.RULES),
"CAMERA AND PROJECTION RULES": Numbered(base.CAMERA_AND_PROJECTION),
- "VLA SEGMENT GATES": Numbered(base.VLA_GATES),
"EXPLORATION WORKFLOW": Numbered(RULES),
"LAYERED MEMORY": MEMORY,
}
diff --git a/robots/dual_franka/prompts/system.py b/robots/dual_franka/prompts/system.py
index ef593641a..60c1b554e 100644
--- a/robots/dual_franka/prompts/system.py
+++ b/robots/dual_franka/prompts/system.py
@@ -41,15 +41,20 @@
"specific historical step.",
"Use purposeful corrections and use the configured inline view(s) as "
"the primary visual evidence.",
+ "For move_delta and rotate_delta, status='completed' means execution "
+ "finished, not exact target attainment. target_reached and residual errors "
+ "are accuracy diagnostics; target_reached=false alone is not a controller "
+ "fault or a requirement to retry, stop, or request operator feedback. "
+ "Use the actual final pose and returned observation for the next action.",
"Never use a VLA trained for another embodiment or action normalization.",
"VLA segment tools accept a planner-facing prompt. A deployment may still "
"override it with a checkpoint-specific training instruction during policy "
"inference; inspect the tool result to see requested/effective prompts.",
"Treat a VLA segment stop as the end of one manipulation segment, not as "
"proof that its physical goal succeeded.",
- "Inspect joint_health after every mutation and recover joint posture before "
- "continuing when either arm reports warning or critical.",
- "If state, cameras, or motion results are inconsistent, stop instead of guessing.",
+ "Stop for actual controller faults, protective motion aborts, or conflicting "
+ "state/camera evidence that prevents safe action. An ordinary residual "
+ "tracking error alone is not such a conflict.",
)
CAMERA_AND_PROJECTION = (
@@ -60,8 +65,9 @@
"Choose a pixel well inside visible material of the named target object or "
"target container, away from silhouettes, rims, walls, wires, occluders, and "
"background. Never project image-space air above an object.",
- "When SAM3 is available, use segment on a registered localization camera to "
- "segment visible target objects or destination regions before manual pixel projection. "
+ "Choose perception tools according to the current task. When using SAM3, "
+ "call segment on a registered localization camera for the intended object "
+ "or destination region. "
"Inspect the returned mask overlay yourself; trust the returned right_base "
"point only when the mask and median marker cover the intended target.",
"SAM3 text prompts are phrase-sensitive. Prefer short color/object/relation "
@@ -75,49 +81,11 @@
"After a verified projection, trust the returned right_base xyz and TCP-to-"
"point deltas as the primary metric evidence; do not override them with "
"unaided 2D RGB distance guesses.",
- "Do not move an arm or held object merely to improve camera visibility. If "
- "visibility is insufficient, mark the state uncertain or stop for operator "
- "feedback instead of performing active-vision motions.",
-)
-
-VLA_GATES = (
- "Use VLA segment tools for contact-rich motion: vla_right_grasp, "
- "vla_handoff, and vla_left_place.",
- "vla_right_grasp ends after right gripper closure plus lift; vla_handoff "
- "ends after right gripper opening plus a configured release delay; "
- "vla_left_place ends after left gripper opening plus lift. These boundaries "
- "only mean a segment ended; verify physical success from images, gripper "
- "widths/open flags, and projection geometry before continuing.",
- "After a grasp VLA, do not decide lifted/held status from 2D appearance "
- "alone. Check right gripper state and the latest inline image. If held status "
- "is unclear, project the best visible held-object surface when possible and "
- "compare right_base z primarily with the source/table height or pre-grasp "
- "source projection.",
- "If a verified projection of the intended object or held-object surface is "
- "higher than source/table height, treat the grasp as successful and proceed "
- "to vla_handoff. Mark failure only when verified projection shows the same "
- "object still at table height and gripper/image evidence shows the right "
- "gripper is empty or not supporting it.",
- "After a confirmed right-hand grasp, call vla_handoff directly. Do not use "
- "move_delta, rotate_delta, open_gripper, close_gripper, or any other "
- "rule-based primitive to prepare or reposition either arm for handoff; the VLA owns the bimanual "
- "approach, relative pose adjustment, left-hand grasp, right-hand release, "
- "and collision-aware transfer.",
- "Before vla_left_place, first verify that vla_handoff has ended and the "
- "left gripper holds the intended object. If the left hand is not clearly "
- "holding the object, do not stage for placement.",
- "For placement staging, project a clearly valid target pixel inside the "
- "required destination, then use projected x/y only. Keep the current left "
- "TCP z with delta_z=0 by default; do not move to the projected z coordinate "
- "or add large vertical clearance. Let vla_left_place handle descent, "
- "release, and post-release lift.",
)
WORKFLOW = (
"Call describe_dual_franka_setup, then inspect the initial synchronized state.",
- "Build a conservative localization table for the current category from the "
- "latest inline image and keep it updated after each primitive.",
"After each motion, inspect the returned result before choosing the next action.",
- "Use the task's VLA segment tools for contact-rich motion.",
+ "Follow the task's policy for choosing VLA skills or direct control.",
"Finish only when the success evidence is visible and consistent with state.",
)
diff --git a/robots/dual_franka/prompts/user.py b/robots/dual_franka/prompts/user.py
index bb78eb4e0..925b45ba6 100644
--- a/robots/dual_franka/prompts/user.py
+++ b/robots/dual_franka/prompts/user.py
@@ -23,6 +23,6 @@
BEGIN = """Call describe_dual_franka_setup before acting. Then call
view_env_state with step 0, inspect the configured inline visual evidence
-together with both arms' TCP/gripper/joint-health state, and execute the task
+together with both arms' TCP/gripper state, and execute the task
conservatively with the exposed bounded tools. Auxiliary camera views are
artifact views for targeted follow-up inspection only."""
diff --git a/robots/dual_franka/robot_spec.py b/robots/dual_franka/robot_spec.py
index aa9dbffc4..6705427b1 100644
--- a/robots/dual_franka/robot_spec.py
+++ b/robots/dual_franka/robot_spec.py
@@ -56,7 +56,7 @@
"name": "task_id",
"kind": "integer",
"minimum": 0,
- "suggestions": tuple(sorted(DUAL_FRANKA_TASKS)),
+ "suggestions": tuple(str(key) for key in sorted(DUAL_FRANKA_TASKS)),
},
),
"display": "Dual Franka task {task_id}",
@@ -424,7 +424,10 @@ def _init_runtime(
connectors = {
"env": lambda rpc: {
"env": DualFrankaEnvClient(
- rpc, reset_on_connect=not getattr(args, "explore", False)
+ rpc,
+ reset_on_connect=not (
+ getattr(args, "explore", False) or getattr(args, "dashboard", False)
+ ),
),
"task_description": get_dual_franka_task(args.task_id).instruction,
"vla_instruction": get_dual_franka_task(args.task_id).vla_instruction,
diff --git a/robots/dual_franka/rpent_env.py b/robots/dual_franka/rpent_env.py
new file mode 100644
index 000000000..5d6a948dd
--- /dev/null
+++ b/robots/dual_franka/rpent_env.py
@@ -0,0 +1,186 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""RPent observations on top of the official RLinf dual-Franka environment."""
+
+from __future__ import annotations
+
+import copy
+import queue
+from dataclasses import replace
+from typing import Any
+
+import cv2
+import gymnasium as gym
+import numpy as np
+from gymnasium.envs.registration import register
+from rlinf.envs.real.franka.dual_franka_tcp import DualFrankaTCPEnv
+from rlinf.envs.real.wrappers import build_stack
+from rlinf.robotics.parts.cameras import Camera, CameraInfo, RealSenseCamera
+
+from robots.franka.rpent_env import realsense_color_intrinsics
+
+_CAMERA_FRAME_TIMEOUT_S = 0.5
+_REALSENSE_BACKEND = "rpent_realsense"
+
+
+@Camera.register(_REALSENSE_BACKEND)
+class RPentRealSenseCamera(RealSenseCamera):
+ """Expose projection metadata from the camera's existing SDK connection."""
+
+ def get_color_intrinsics(self) -> dict[str, Any]:
+ """Read serializable intrinsics on the node that owns the camera."""
+ return realsense_color_intrinsics(self)
+
+
+class RPentDualFrankaEnv(DualFrankaTCPEnv):
+ """Preserve policy observations and retain their uncropped RGB-D for RPent."""
+
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ self._rpent_camera_snapshot: dict[str, dict[str, np.ndarray]] | None = None
+ super().__init__(*args, **kwargs)
+
+ def _camera_infos(self) -> list[CameraInfo]:
+ # A public method on this driver is also forwarded by RLinf's remote
+ # camera handle; SDK profile objects themselves cannot cross that boundary.
+ return [
+ replace(info, camera_type=_REALSENSE_BACKEND)
+ if info.camera_type == "realsense"
+ else info
+ for info in super()._camera_infos()
+ ]
+
+ def _get_camera_observation(
+ self,
+ ) -> tuple[dict[str, np.ndarray], dict[str, np.ndarray]]:
+ """Collect once for policy and perception, invalidating stale raw data."""
+ self._rpent_camera_snapshot = None
+ frames: dict[str, np.ndarray] = {}
+ depths: dict[str, np.ndarray] = {}
+ raw_frames: dict[str, np.ndarray] = {}
+ raw_depths: dict[str, np.ndarray] = {}
+ display_frames: dict[str, np.ndarray] = {}
+ all_fresh = True
+
+ for name, camera in self._cameras.items():
+ try:
+ reading = camera.get_observation(
+ timeout=_CAMERA_FRAME_TIMEOUT_S,
+ max_age=_CAMERA_FRAME_TIMEOUT_S,
+ )
+ except queue.Empty:
+ # Preserve RLinf's policy fallback during camera recovery, but
+ # never present its old image as a fresh projection observation.
+ cached = self._last_camera_frame.get(name)
+ if cached is None:
+ raise RuntimeError(
+ f"Camera {name} stalled with no cached frame to fall back to."
+ ) from None
+ self._logger.error("Camera %s stalled; using the last frame.", name)
+ reading = cached
+ all_fresh = False
+
+ raw_frames[name] = reading["frame"][..., ::-1].copy()
+ reshape_size = self.observation_space["frames"][name].shape[:2][::-1]
+ cropped, resized = self._crop_frame(reading["frame"], reshape_size)
+ frames[name] = resized[..., ::-1]
+ display_frames[name] = resized
+ display_frames[f"{name}_full"] = cropped
+ if "depth" in reading:
+ raw_depths[name] = reading["depth"].copy()
+ _, depths[name] = self._crop_frame(
+ reading["depth"], reshape_size, interpolation=cv2.INTER_NEAREST
+ )
+ self._last_camera_frame[name] = reading
+
+ self.camera_player.put_frame(display_frames)
+ if all_fresh:
+ self._rpent_camera_snapshot = {
+ "raw_frames": raw_frames,
+ "raw_depths": raw_depths,
+ }
+ return frames, depths
+
+ def get_raw_camera_metadata(self) -> dict[str, dict[str, Any]]:
+ """Describe the complete, uncropped snapshot and supported calibration."""
+ snapshot = self._rpent_camera_snapshot
+ if snapshot is None:
+ raise RuntimeError(
+ "No fresh complete camera snapshot; acquire an observation first."
+ )
+ metadata = {}
+ for name, camera in self._cameras.items():
+ info = camera.camera_info
+ depth = snapshot["raw_depths"].get(name)
+ intrinsics = (
+ camera.get_color_intrinsics()
+ if isinstance(camera, RPentRealSenseCamera)
+ else None
+ )
+ metadata[name] = {
+ "name": name,
+ "serial_number": info.serial_number,
+ "camera_type": (
+ "realsense"
+ if info.camera_type == _REALSENSE_BACKEND
+ else info.camera_type
+ ),
+ "rgb_shape": list(snapshot["raw_frames"][name].shape),
+ "depth_shape": None if depth is None else list(depth.shape),
+ "depth_enabled": info.enable_depth,
+ "depth_available": depth is not None,
+ "depth_unit": None if depth is None else "m",
+ "depth_scale": float(camera.depth_scale),
+ "color_intrinsics": intrinsics,
+ "projection_supported": depth is not None and intrinsics is not None,
+ }
+ return metadata
+
+ def get_raw_camera_snapshot(self) -> dict[str, Any]:
+ """Return a copy of the preceding acquisition without reading again."""
+ metadata = self.get_raw_camera_metadata()
+ return {
+ **copy.deepcopy(self._rpent_camera_snapshot),
+ "camera_meta": metadata,
+ }
+
+
+def create_rpent_dual_franka_env(
+ override_cfg: dict,
+ worker_info: object,
+ robot_info: object,
+ env_idx: int,
+ env_cfg: dict,
+) -> gym.Env:
+ """Create the dual-Franka adapter with RLinf's standard wrapper stack."""
+ env = RPentDualFrankaEnv(
+ override_cfg=override_cfg,
+ worker_info=worker_info,
+ robot_info=robot_info,
+ env_idx=env_idx,
+ )
+ try:
+ return build_stack(env, env_cfg)
+ except BaseException:
+ env.close()
+ raise
+
+
+def register_rpent_dual_franka_env() -> None:
+ """Register the RPent dual-Franka adapter with Gymnasium."""
+ if "RPentDualFrankaTCPEnv-v1" not in gym.registry:
+ register(
+ id="RPentDualFrankaTCPEnv-v1",
+ entry_point="robots.dual_franka.rpent_env:create_rpent_dual_franka_env",
+ )
diff --git a/robots/dual_franka/rpent_live_env.sh b/robots/dual_franka/rpent_live_env.sh
index 36ada25cf..07aafaa6c 100644
--- a/robots/dual_franka/rpent_live_env.sh
+++ b/robots/dual_franka/rpent_live_env.sh
@@ -17,9 +17,6 @@ _RPENT_LIVE_ENV_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export RPENT_REPO_ROOT="${RPENT_REPO_ROOT:-$(cd "${_RPENT_LIVE_ENV_SCRIPT_DIR}/../.." && pwd)}"
export RPENT_TEST_ROOT="${RPENT_TEST_ROOT:-$(cd "${RPENT_REPO_ROOT}/.." && pwd)}"
-# External checkout used by RPent/Ray workers.
-export RLINF_REPO_PATH="${RLINF_REPO_PATH:-${RPENT_TEST_ROOT}/RLinf}"
-
# Isolate Codex records/config/cache/memory used by robot-control sessions from
# the user's normal assistant-for-coding Codex state.
export CODEX_HOME="${RPENT_CODEX_HOME:-${RPENT_REPO_ROOT}/.codex-rpent-live}"
@@ -37,10 +34,10 @@ if [[ -n "${RPENT_CODEX_BASE_URL:-}" ]]; then
export CODEX_BASE_URL="${RPENT_CODEX_BASE_URL}"
fi
-if [[ -n "${PYTHONPATH:-}" ]]; then
- export PYTHONPATH="${RLINF_REPO_PATH}:${RPENT_REPO_ROOT}:${PYTHONPATH}"
-else
- export PYTHONPATH="${RLINF_REPO_PATH}:${RPENT_REPO_ROOT}"
+# Use the installed RLinf package unless a developer explicitly selects a checkout.
+export PYTHONPATH="${RPENT_REPO_ROOT}${PYTHONPATH:+:${PYTHONPATH}}"
+if [[ -n "${RLINF_REPO_PATH:-}" ]]; then
+ export PYTHONPATH="${RLINF_REPO_PATH}:${PYTHONPATH}"
fi
# Shared live service endpoints. Override these before sourcing, or set them on
diff --git a/robots/dual_franka/run_dual_franka_interactive.sh b/robots/dual_franka/run_dual_franka_interactive.sh
index 1fe1d0ac1..b5c34ed95 100755
--- a/robots/dual_franka/run_dual_franka_interactive.sh
+++ b/robots/dual_franka/run_dual_franka_interactive.sh
@@ -19,6 +19,7 @@ exec .venv/bin/python -m rpent.cli.main \
--model "${RPENT_CODEX_MODEL:-gpt-5.5}" \
--reasoning-effort "${RPENT_REASONING_EFFORT:-medium}" \
--interactive \
+ --planner-timeout-s 0 \
--memory-profile local \
--memory-dir "${RPENT_LIVE_MEMORY_DIR}" \
--env-endpoint "${RPENT_ENV_ENDPOINT}" \
diff --git a/robots/dual_franka/runtime_config.py b/robots/dual_franka/runtime_config.py
index 2bda35110..2541c22c8 100644
--- a/robots/dual_franka/runtime_config.py
+++ b/robots/dual_franka/runtime_config.py
@@ -133,19 +133,6 @@ def _projection_views(raw: dict[str, Any]) -> dict[str, Any]:
}
-def _joint_health_thresholds(raw: dict[str, Any]) -> dict[str, dict[str, float]]:
- """Load per-arm joint-health thresholds from the user robot config."""
- joint_health = _require_mapping(raw.get("joint_health"), "joint_health")
- thresholds = _require_mapping(
- joint_health.get("thresholds"), "joint_health.thresholds"
- )
- out: dict[str, dict[str, float]] = {}
- for arm in ("left", "right"):
- values = _require_mapping(thresholds.get(arm), f"joint_health.thresholds.{arm}")
- out[arm] = {str(key): float(value) for key, value in values.items()}
- return out
-
-
def load_runtime_config(
path: str | Path | None,
*,
@@ -169,7 +156,6 @@ def load_runtime_config(
cameras = _require_mapping(raw.get("cameras"), "cameras")
observation = _require_mapping(cameras.get("observation"), "cameras.observation")
workspace = _require_mapping(raw.get("workspace"), "workspace")
- joint_health_thresholds = _joint_health_thresholds(raw)
base_serials, base_type = _camera_slot(observation, "base")
left_serials, left_type = _camera_slot(observation, "left_wrist")
@@ -194,6 +180,15 @@ def load_runtime_config(
"left_controller_node_rank": LEFT_CONTROLLER_NODE,
"right_controller_node_rank": RIGHT_CONTROLLER_NODE,
"node_rank": HARDWARE_NODE,
+ **(
+ {
+ "compliance": _require_mapping(
+ robot["compliance"], "robot.compliance"
+ )
+ }
+ if "compliance" in robot
+ else {}
+ ),
},
where="cluster.node_groups[].hardware.configs[]",
)
@@ -241,21 +236,71 @@ def load_runtime_config(
"keyboard_reward_wrapper": None,
"use_relative_frame": False,
"video_cfg": {},
- "init_params": {"id": "DualFrankaTCPEnv-v1"},
+ "init_params": {"id": "RPentDualFrankaTCPEnv-v1"},
"override_cfg": override_cfg,
}
},
}
)
controller = flatten_control(CONTROL)
+ rotation_hold = _require_mapping(raw.get("rotation_hold", {}), "rotation_hold")
+ position_tolerance = float(
+ rotation_hold.get("tolerance_m", CONTROL["move"]["tolerance_m"])
+ )
+ max_drift = float(rotation_hold.get("max_drift_m", 0.03))
+ if not 0 < position_tolerance < max_drift < float("inf"):
+ raise ValueError(
+ "rotation_hold requires 0 < tolerance_m < max_drift_m < infinity"
+ )
+ controller["rotate_position_tolerance_m"] = position_tolerance
+ controller["rotate_max_drift_m"] = max_drift
+ integral_gain = float(rotation_hold.get("integral_gain_per_s", 0.5))
+ integral_limit = float(rotation_hold.get("integral_limit_rad", 0.12))
+ if not 0 <= integral_gain < float("inf") or not 0 <= integral_limit <= 0.3:
+ raise ValueError(
+ "rotation_hold integral gain must be finite/nonnegative and limit in [0, 0.3] rad"
+ )
+ controller["rotate_integral_gain_per_s"] = integral_gain
+ controller["rotate_integral_limit_rad"] = integral_limit
+ position_integral_gain = float(
+ rotation_hold.get("position_integral_gain_per_s", 0.5)
+ )
+ position_integral_limit = float(
+ rotation_hold.get("position_integral_limit_m", 0.015)
+ )
+ if (
+ not 0 <= position_integral_gain < float("inf")
+ or not 0 <= position_integral_limit < max_drift
+ ):
+ raise ValueError(
+ "rotation_hold position integral gain must be finite/nonnegative and limit below max_drift_m"
+ )
+ controller["rotate_position_integral_gain_per_s"] = position_integral_gain
+ controller["rotate_position_integral_limit_m"] = position_integral_limit
+ settle_s = float(rotation_hold.get("settle_s", 0.5))
+ if not 0 <= settle_s < controller["rotate_timeout_s"]:
+ raise ValueError(
+ "rotation_hold.settle_s must be nonnegative and below rotate timeout"
+ )
+ controller["rotate_settle_s"] = settle_s
controller["robot_config_path"] = str(
Path(path or DEFAULT_CONFIG).expanduser().resolve()
)
controller["perception"] = _perception_cameras(cameras)
controller["agent_observation"] = _agent_observation(cameras)
controller["projection_views"] = _projection_views(raw)
+ health = _require_mapping(raw.get("joint_health"), "joint_health")
+ thresholds = _require_mapping(health.get("thresholds"), "joint_health.thresholds")
+ controller["joint_health_thresholds"] = {
+ arm: {
+ str(key): float(value)
+ for key, value in _require_mapping(
+ thresholds.get(arm), f"joint_health.thresholds.{arm}"
+ ).items()
+ }
+ for arm in ("left", "right")
+ }
controller["recovery_return_timeout_s"] = RECOVERY["return_timeout_s"]
controller["recovery_return_tolerance_m"] = RECOVERY["return_tolerance_m"]
controller["recovery_return_tolerance_rad"] = RECOVERY["return_tolerance_rad"]
- controller["joint_health_thresholds"] = joint_health_thresholds
return FrankaRuntimeConfig(rlinf=rlinf, controller=controller)
diff --git a/robots/dual_franka/tasks.py b/robots/dual_franka/tasks.py
index 5dd282e09..8d0f7f2bc 100644
--- a/robots/dual_franka/tasks.py
+++ b/robots/dual_franka/tasks.py
@@ -150,7 +150,6 @@
"Before calling vla_left_place, first confirm vla_handoff has ended and the left gripper is holding the object; then use the latest inline D455 image to localize the required placement target; prefer segment with camera='d455' for the target region when SAM3 is available, otherwise call back_project with camera='d455' on an interior pixel of that placement target; use only safe left-arm free-space move_delta commands to horizontally align the left-held object over the projected right_base target when staging is needed. Use the projected target x/y only; keep the current left TCP z with delta_z=0 by default, do not move to the projected z coordinate, and do not add a large vertical clearance. Only use a tiny safety lift if the current carried object is visibly below the rim or at collision risk. Call vla_left_place only if the latest staging move succeeded and reports a reached/effective target. If the staging move fails, is uncertain, or does not reach the target, do not call vla_left_place.",
"Named VLA skills may end early through semantic boundary rules: grasp ends after right gripper closure plus lift, handoff ends after the right gripper opens and the configured release delay elapses, and place ends after the left gripper opens and then lifts about 10cm. This only means the current skill segment ended; inspect images and grippers before judging whether it succeeded.",
"When calling a named grasp, handoff, or placement VLA skill, usually leave max_chunks unset so the server gives the skill enough budget and stops it at the semantic boundary. Override max_chunks only for a cautious diagnostic run.",
- "After every VLA or rule-based action, inspect joint_health in the returned snapshot. If either arm is warning or critical, especially right-arm q1 positive / q3 negative accumulation, call recover_joint_posture before continuing more VLA chunks; it records each gripper's open/closed state, re-commands that state before and after joint reset so closed grippers stay clamped, resets both arms' joints, then returns both TCPs near their pre-recovery right_base poses.",
OPERATOR_FINISH_CONSTRAINT,
"If depth is missing, projection is uncertain, the intended object is occluded, or success is ambiguous, do not infer metric deltas from RGB alone; stop for operator feedback instead of opening a gripper or blindly retrying.",
),
@@ -240,9 +239,9 @@
"The metal wire basket/frame is the target only for dirty bowls and dirty plates. The black-outside, white-inside cardboard box is the target only for clean bowls and clean plates. Do not count dirty objects in the cardboard box or clean objects in the metal basket as success.",
"Advance to the next category only after the latest inline D455 image confirms that every object in the current category has reached its required destination, or that no such object remains available in the source workspace after a careful visual check.",
"The D455 is a fixed external camera. Do not move either arm or held object merely to improve camera visibility. Arm motion is allowed only for task execution or safety: free-space approach/staging, VLA handoff/placement/grasp execution, or joint recovery.",
- "After every VLA or rule-based action, inspect joint_health in the returned snapshot. If either arm is warning or critical, especially right-arm q1 positive / q3 negative accumulation, call recover_joint_posture before continuing more VLA chunks; it records each gripper's open/closed state, re-commands that state before and after joint reset so closed grippers stay clamped, resets both arms' joints, then returns both TCPs near their pre-recovery right_base poses.",
OPERATOR_FINISH_CONSTRAINT,
"If depth is missing, projection is uncertain, the intended object is occluded, dirty/clean status is uncertain, or success is ambiguous, do not infer metric deltas from RGB alone; stop for operator feedback instead of opening a gripper or blindly retrying.",
+ "vla_right_grasp ends after right gripper closure plus lift; vla_handoff ends after right gripper opening plus the configured release delay; vla_left_place ends after left gripper opening plus lift. These boundaries only mean a skill segment ended, not that the intended object was successfully grasped, transferred, or placed. Verify the intended object and physical outcome from the returned images and gripper state before continuing; use projection evidence when needed.",
),
),
}
@@ -304,7 +303,7 @@
_DIRTY_CLEAN_CONSTRAINTS[20],
_DIRTY_CLEAN_CONSTRAINTS[21],
_DIRTY_CLEAN_CONSTRAINTS[22],
- _DIRTY_CLEAN_CONSTRAINTS[23],
+ "vla_right_grasp ends after right gripper closure plus lift; vla_handoff ends after right gripper opening plus the configured release delay; vla_left_place ends after left gripper opening plus lift. These boundaries only mean a skill segment ended, not that the intended object was successfully grasped, transferred, or placed. Verify the intended object and physical outcome from the returned images and gripper state before continuing; use projection evidence when needed.",
),
)
@@ -321,10 +320,10 @@
"basket/frame. Treat the two chopsticks as two separate objects to "
"process one at a time; lifting one chopstick while another chopstick "
"remains on the table is valid progress, not a failed pair grasp. The "
- "two chopsticks and the spoon also go into the metal wire basket/frame, "
- "not into the cup. The cup should be placed upright in the metal basket "
- "when feasible, but the cup does not need to support later utensil "
- "insertion. Every object must go through one right-to-"
+ "two chopsticks go into the metal wire basket/frame. The spoon must "
+ "be inserted into the cup; no other destination is acceptable. Place "
+ "the cup upright and stable inside the metal basket before inserting "
+ "the spoon. Every object must go through one right-to-"
"left handoff: the right hand grasps the object, vla_handoff transfers "
"it to the left hand, and the left hand places it with vla_left_place."
),
@@ -341,51 +340,125 @@
"depth can be back-projected into the right_base frame using the "
"calibrated extrinsics. The D455 is a fixed external camera; moving "
"either arm or a held object does not move the camera viewpoint and "
- "must not be used as an active-vision strategy. right_base is the "
+ "is not itself a camera-viewpoint change. Object/arm adjustments may "
+ "change visibility or depth against a different background. right_base is the "
"shared world coordinate frame exposed to the agent: camera points, "
"left/right TCP positions, and rule-based base-frame deltas are all "
"represented in right_base. The VLA checkpoint is the deployment-"
"aligned dual-arm tcp-rot6d clean-desk policy; named VLA tools keep "
"using its fixed training instruction, while the planner chooses the "
- "metal basket as the placement target."
+ "metal basket as the placement target for bowls, plates, the cup and "
+ "chopsticks, and the cup interior as the only spoon placement target."
),
success_criteria=(
"All target objects have been processed. Every bowl, every plate, and the cup are clearly inside the "
"open interior of the metal wire basket/frame. The cup is upright and "
- "stable inside the metal basket when feasible. Each individual "
- "chopstick and the spoon are also inside the metal wire basket/frame. No target "
+ "stable inside the metal basket. Each individual chopstick is inside "
+ "the metal wire basket/frame. The spoon is inserted into the cup. "
+ "A spoon lying in a bowl, on a plate, loose in the basket, or anywhere "
+ "outside the cup does not satisfy the task. No target "
"object remains in the source workspace, in either gripper, on the "
- "table near the containers, or in the cardboard box. Stop for "
- "operator feedback if an object was placed in the wrong container or "
- "if visual evidence does not support safe continuation."
+ "table near the containers, or in the cardboard box. A wrong placement "
+ "is unfinished: inspect it and recover autonomously when safely possible. "
+ "Ask for operator assistance only when a concrete blocker requires human "
+ "intervention or no evidence-supported safe continuation remains."
),
constraints=(
"Call describe_dual_franka_setup before acting.",
"Call view_env_state before the first action only if no fresh primitive snapshot is already available. After any primitive that returns a snapshot, inspect that returned snapshot directly; never call view_env_state immediately afterward unless the primitive failed, returned no snapshot, an operator changed the scene, or a specific historical step is needed.",
"Enforce the required object order without exception: bowls -> plates -> cup -> chopsticks -> spoon. Always act on the earliest unfinished category. While any object from that category remains in the source workspace, do not localize, approach, grasp, or call a VLA grasp skill for a later category. Within bowls, plates, and chopsticks, process the green object before the blue object; do not act on the blue object while the green object from the same category remains unfinished.",
"Do not perform dirty/clean classification in this task. Egg tarts or foil trays, if visible, are irrelevant to the destination decision. Bowls, plates, and the cup all have the same destination: the open interior of the metal wire basket/frame.",
- "Build a brief D455 localization table before choosing the next object. For each visible candidate, list object type, color if visible, required_destination, D455 evidence, projected point if used, and status: source_workspace, inside_metal_basket, cardboard_box_wrong_destination, or uncertain.",
+ "Build a brief D455 localization table before choosing the next object. For each visible candidate, list object type, color if visible, required_destination, D455 evidence, projected point if used, and status: source_workspace, held, inside_metal_basket, in_cup, wrong_destination, or uncertain. For the spoon, inside_metal_basket without in_cup is a wrong destination, not completion.",
"The metal wire basket/frame is the only valid placement container for bowls, plates, and the cup. The black-outside, white-inside cardboard box is not a destination in this task. Never count objects in the cardboard box, on the table near the containers, touching the outside of the metal wires, or ambiguous between the two containers as success.",
- "When SAM3 is available, use segment with camera='d455' on the D455 image to segment the current object or required placement target before falling back to a manual pixel. Use short phrases, inspect the returned mask overlay yourself, and retry with a point prompt, more specific text prompt, or manual D455 pixel whenever the mask or median marker is not visibly on the intended target.",
- "Use back_project only on a pixel well inside the visible material of the intended object or the open metal basket interior. Do not select the table, background, rim edge, air above the object, another object, a container wall, metal wire, the cup as a utensil destination, or a shared container boundary. Inspect the returned annotated image; if the marker is not on the intended target, retry or stop.",
- "When exact spatial relation is uncertain, use segment or back_project again instead of judging metric x/y/z offsets by 2D RGB appearance. After a segmentation/projection is verified, trust the returned right_base xyz and TCP-to-point deltas as the main metric evidence for rule-based correction moves.",
+ "Choose available camera views and perception tools as useful for the current object or required placement target. When using SAM3 on D455, use segment with camera='d455', use short phrases, and inspect the returned mask overlay. If the mask or median marker is not visibly on the intended target, revise the prompt, use a point prompt or manual pixel, or choose another useful observation.",
+ "When using back_project, select a pixel well inside visible material of the intended object or its required destination. For spoon placement, the cup interior is the required projection target; for bowls, plates, the cup, and chopsticks, use the open metal basket interior. Do not select the table, background, rim edge, air, another object, a container wall, metal wire, or a shared container boundary. Inspect the returned annotated image before using its coordinates; a misplaced marker is not valid target evidence.",
+ "Decide whether fresh segmentation or projection would help resolve an uncertain spatial relation. RGB-based visual control is allowed and does not require a successful depth projection first. Do not present RGB estimates or invalid depth as calibrated metric measurements. After a segmentation/projection is verified, use the returned right_base xyz and TCP-to-point deltas as metric evidence for rule-based correction moves.",
"Treat right_base as the only world coordinate frame for agent reasoning. For both left and right arms, move_delta and rotate_delta use right_base/world deltas; do not convert left-arm coordinates yourself.",
"For bowls, do not use rule-based move_delta to pre-align or move above the bowl before vla_right_grasp. This avoids pushing the learned VLA policy out of distribution. After D455 confirms the intended bowl identity, call vla_right_grasp directly from the current robot state.",
"For plates, the cup, chopsticks, and the spoon, rule-based pre-grasp staging is allowed only when clearly needed: move the right TCP to a safe pre-grasp pose about 10cm above the projected target surface point in right_base/world coordinates, without touching the object.",
"Call vla_right_grasp only after the intended object identity is confirmed. For non-bowl objects, call vla_right_grasp only after the latest right-arm move_delta reports a reached/effective target near the intended object, unless the object is already safely staged. Leave max_chunks unset unless running a cautious diagnostic.",
"For plates specifically, a failed grasp or drop is recoverable: do not stop immediately just because the plate is still on the table. If the same plate remains visible, reachable, and safe, retry that plate after re-localizing and safe staging. Do not bypass the required category/color order by choosing a later object.",
"For chopsticks specifically, process one visible chopstick at a time. Do not require grasping both chopsticks together. After a chopstick grasp VLA, inspect the right_wrist artifact as primary evidence for whether the current chopstick is held, because D455 may still show the other chopstick on the table or may miss a thin held chopstick. If right_wrist shows one chopstick clamped or supported by the right gripper and the gripper is closed, treat that individual chopstick grasp as successful and proceed to vla_handoff; treat the remaining table chopstick as a separate unfinished object that can be handled later.",
- "Use VLA segment tools for contact-rich motion: vla_right_grasp for the right-arm grasp segment, vla_handoff for bimanual transfer, and vla_left_place for left-arm placement into the metal basket.",
- "After a grasp VLA, inspect the returned snapshot. Continue only if the right gripper appears to hold the intended object. Do not judge lifted/held status from 2D image appearance alone. If it is unclear whether the object has lifted or remains on the table, call back_project with camera='d455' on the best visible held-object surface when possible. If only a source/table candidate is visible, treat that projection as a source-candidate check, not proof of failure. Compare returned right_base z primarily with the source/table height or pre-grasp source projection. For wide objects such as plates, projected center/rim x/y can be far from the right TCP while still held; do not mark failure from TCP x/y distance alone. If a verified projection of the intended object or held-object surface is higher than the source/table height or pre-grasp source projection, treat the grasp as successful and proceed to vla_handoff. Mark failure only when a verified projection shows the same intended object still at source/table height and gripper/image evidence shows the right gripper is empty or not supporting it. If a plate grasp fails or the plate drops back onto the table, keep the category gate on that same plate, verify from D455 that the plate is safely reachable and not entangled with other objects, reopen/recover the right gripper if needed, re-localize the same plate, and retry vla_right_grasp instead of stopping immediately. Stop for operator feedback only if the plate identity is uncertain, the plate is occluded, unsafe, unreachable, entangled, or repeated retry evidence shows the same unsafe failure mode. If the wrong object is grasped or identity is uncertain, stop for operator feedback.",
+ "Use VLA segment tools for contact-rich motion: vla_right_grasp for the right-arm grasp segment, vla_handoff for bimanual transfer, and vla_left_place for left-arm placement at the object's required destination: the cup interior for the spoon, the metal basket for other objects.",
+ "After a grasp VLA, inspect the returned snapshot. Continue only if the right gripper appears to hold the intended object. Do not judge lifted/held status from 2D image appearance alone. If projection would help determine whether the object has lifted or remains on the table, use back_project with camera='d455' on the best visible held-object surface. If only a source/table candidate is visible, treat that projection as a source-candidate check, not proof of failure. Compare returned right_base z primarily with the source/table height or pre-grasp source projection. For wide objects such as plates, projected center/rim x/y can be far from the right TCP while still held; do not mark failure from TCP x/y distance alone. If a verified projection of the intended object or held-object surface is higher than the source/table height or pre-grasp source projection, treat the grasp as successful and proceed to vla_handoff. Mark failure only when a verified projection shows the same intended object still at source/table height and gripper/image evidence shows the right gripper is empty or not supporting it. If a plate grasp fails or the plate drops back onto the table, keep the category gate on that same plate, verify from D455 that the plate is safely reachable and not entangled with other objects, reopen/recover the right gripper if needed, re-localize the same plate, and retry vla_right_grasp instead of stopping immediately. If identity or support is uncertain, first inspect current alternative camera views and use the perception-recovery options below. An incorrect grasp is unfinished, not automatically a failed attempt. Request operator help only if useful checks cannot resolve identity/support, the object is unsafe, unreachable or entangled, or retries show the same unsafe failure mode.",
"After a confirmed right-hand grasp, call vla_handoff directly. Do not call move_delta, rotate_delta, open_gripper, close_gripper, or any other rule-based primitive to prepare or reposition either arm before handoff; vla_handoff owns the complete bimanual approach and transfer from the post-grasp state.",
- "Before vla_left_place, confirm vla_handoff has ended and the left gripper holds the intended object. For bowls and plates, localize an interior placement point inside the open metal wire basket/frame and use the same placement method as before: stage the left-held object with safe left-arm horizontal x/y move_delta only when staging is needed, use the projected target x/y, keep the current left TCP z with delta_z=0 by default, do not move to the projected z coordinate, and do not add a large vertical clearance. Call vla_left_place only if the latest staging move succeeded and reports a reached/effective target. If the staging move fails, is uncertain, or does not reach the target, do not call vla_left_place.",
- "For the cup, localize an interior placement point inside the open metal basket and use the same strict staging gate as bowls and plates. Prefer an upright stable placement when feasible, but do not require the cup to support later utensil insertion.",
- "For each chopstick and for the spoon, localize an interior placement point inside the open metal basket, stage the left-held utensil above that basket point using safe free-space x/y alignment, and call vla_left_place only if the latest staging move succeeded and reports a reached/effective target. The utensil destination is the metal basket, not the cup or the cardboard box.",
- "After each placement, use the latest inline D455 image and relevant wrist artifacts to update which objects are complete. Advance to the next category only after every object in the current category is confirmed inside the metal basket, and continue in the required category/color order until every bowl, plate, the cup, each chopstick, and the spoon are inside the metal basket.",
- "The D455 is a fixed external camera. Do not move either arm or held object merely to improve camera visibility. Arm motion is allowed only for task execution or safety: free-space approach/staging, VLA handoff/placement/grasp execution, or joint recovery.",
- "After every VLA or rule-based action, inspect joint_health in the returned snapshot. If either arm is warning or critical, especially right-arm q1 positive / q3 negative accumulation, call recover_joint_posture before continuing more VLA chunks; it records each gripper's open/closed state, re-commands that state before and after joint reset so closed grippers stay clamped, resets both arms' joints, then returns both TCPs near their pre-recovery right_base poses.",
+ "Put bowls and plates into the metal basket.",
+ "Put the cup into the metal basket, upright and stable for subsequent spoon insertion.",
+ "Put the chopsticks into the metal basket. Insert the spoon into the cup; no other destination is acceptable. A spoon in a bowl, on a plate, loose in the basket, or outside the cup is unfinished, not success.",
+ "After each placement, use the latest inline D455 image and relevant wrist artifacts to update which objects are complete. Advance to the next category only after every object in the current category is confirmed at its required destination. Continue in the required category/color order until every bowl, plate, the cup, and each chopstick are inside the metal basket and the spoon is inserted into the cup.",
+ "The D455 is fixed. Choose observation sources, motion direction, distance and strategy from the current scene; this task imposes no small-motion requirement or fixed adjustment sequence. Arm/object motion can change occlusion or background, but does not move D455 itself.",
OPERATOR_FINISH_CONSTRAINT,
- "If depth is missing, projection is uncertain, the intended object is occluded, placement success is ambiguous, or success is otherwise ambiguous, do not infer metric deltas from RGB alone; stop for operator feedback instead of opening a gripper or blindly retrying.",
+ "Missing depth or ambiguous placement alone is not a reason to request operator feedback. Decide how to perceive, adjust and continue using available images and robot state; RGB-based control is allowed. Invalid depth is not a valid metric measurement. Request operator input for a concrete blocker requiring human assistance or a safety issue, not routine uncertainty. Keep final success/failure evaluation and scene-reset confirmation mandatory.",
+ "vla_right_grasp ends after right gripper closure plus lift; vla_handoff ends after right gripper opening plus the configured release delay; vla_left_place ends after left gripper opening plus lift. These boundaries only mean a skill segment ended, not that the intended object was successfully grasped, transferred, or placed. Verify the intended object and physical outcome from the returned images and gripper state before continuing; use projection evidence when needed.",
+ "After vla_handoff ends, verify that the left gripper actually holds the intended object before calling vla_left_place. If the transfer failed or the held object is uncertain, resolve that state before attempting placement.",
+ "Keep the D455 localization table updated after each primitive result, including which objects remain in the source workspace, are held, or are confirmed at their required destination. Track the spoon as complete only when inserted into the cup, not merely inside the metal basket.",
+ ),
+)
+
+
+DUAL_FRANKA_TASKS[6] = replace(
+ DUAL_FRANKA_TASKS[4],
+ name="clean_desk_dirty_clean_sorting_two_cups_explore_candidate",
+ instruction=(
+ "Complete the dirty/clean tabletop sorting task with exactly two cups. "
+ "Process bowls, plates, then both cups; finish each "
+ "category before starting the next. Within a category, process green "
+ "before blue when applicable. Classify bowls using chicken nuggets "
+ "inside them, and plates and cups using an egg-tart foil tray marker: "
+ "marked objects are dirty and go into "
+ "the metal wire basket/frame; unmarked objects are clean and go into "
+ "the black-outside, white-inside cardboard box. Handle both cups "
+ "individually, not as one object. The right hand grasps each object, "
+ "transfers it to the left hand, and the left hand places "
+ "it at its classified destination. Only bowls, plates, and the two "
+ "cups are task objects. There are no chopsticks or spoons to process."
+ ),
+ setup=(
+ "This is the task 4 sorting scene with one additional cup, for exactly "
+ "two cups in total. Do not assume either cup's classification from its "
+ "color or from the other cup. Chicken nuggets inside a bowl mark "
+ "that bowl as dirty; the bowl marker is not an egg tart or foil tray. "
+ "An egg-tart foil tray visibly inside a cup or on a plate's concave "
+ "eating surface marks that specific cup or plate as dirty. Absence of the marker counts "
+ "as clean only when that object's interior is sufficiently visible. "
+ "The adjacent metal wire basket/frame receives dirty objects; the "
+ "black-outside, white-inside cardboard box receives clean objects. "
+ "The cardboard box is on the +y side of the metal basket in right_base. "
+ "Use D455 as the primary observation and calibrated metric localization "
+ "source. D455 is fixed; wrist cameras provide RGB evidence. All exposed "
+ "TCP positions, projected points, and base-frame motion deltas use "
+ "right_base/world. Named VLA skills retain the deployed clean-desk "
+ "policy instruction. Choosing a destination in planner text does not "
+ "retarget that policy: its demonstrated left-placement destination is "
+ "the metal basket. For a clean object's cardboard-box destination, "
+ "use a suitable recorded rule-based placement or plan direct-control "
+ "placement from current perception; changing the VLA prompt is not "
+ "a substitute for those actions."
+ ),
+ success_criteria=(
+ "All bowls, plates, and both cups have been individually classified "
+ "and placed: every dirty object is inside the metal wire basket/frame "
+ "and every clean object is inside the cardboard box's white interior. "
+ "Both cups are accounted for separately at their correct destinations. "
+ "No target object remains at source or in a gripper. "
+ "Uncertain classification, a missing cup, or a wrong destination does "
+ "not count as success. Final success requires operator confirmation."
+ ),
+ constraints=(
+ *DUAL_FRANKA_TASKS[4].constraints[:2],
+ "Process bowls -> plates -> both cups. Both cups must be completed. Within a category, green precedes blue when applicable; dirty/clean status must not reorder grasping.",
+ "Classify each bowl, plate, and cup from current evidence before choosing its placement destination. Chicken nuggets inside a bowl mark it dirty. An egg-tart foil tray inside a cup or on a plate's eating surface marks that cup or plate dirty. A marker nearby or in another object does not classify the chosen object. An occluded interior alone is not evidence of cleanliness.",
+ "Choose available perception tools and camera views as useful. Inspect segmentation overlays and projected markers before using their coordinates; invalid depth is not a valid metric measurement.",
+ DUAL_FRANKA_TASKS[4].constraints[9],
+ "Choose VLA skills, direct control, or a combination for grasping, handoff, placement, and contact-rich adjustments. Contact is not restricted to VLA, and direct-control actions are not restricted to free-space staging. Verify the actual grasp, transfer, and placement outcomes from returned images and state.",
+ "Judge motion accuracy by what the next manipulation needs, not by eliminating every residual error. If the actual pose and visual evidence support continuing, proceed even when target_reached is false. Do not repeatedly micro-adjust or request feedback solely to satisfy a numerical tolerance. Choose whether and how much to correct from the current object, destination, clearance, and observed outcome; no fixed correction size or mandatory alignment is imposed. This does not waive actual controller faults or protective aborts, and action completion is not proof of task success.",
+ "Before placement, verify that the left gripper holds the intended object. Choose its destination: dirty bowls, plates, and cups go to the metal basket; clean bowls, plates, and cups go to the cardboard box.",
+ "Do not count dirty objects in the cardboard box or clean objects in the metal basket as success. Track the two cups separately by current visual identity and classification; placing one does not complete the cup category.",
+ DUAL_FRANKA_TASKS[4].constraints[17],
+ "D455 is fixed, but arm/object motion can improve visibility or change the background. Choose observation motions, direction, distance, and strategy from the scene; no small-motion requirement or fixed adjustment sequence is imposed.",
+ OPERATOR_FINISH_CONSTRAINT,
+ "Missing depth or uncertain localization, classification, or placement alone does not require stopping for operator feedback. Use available perception, RGB evidence, motion, and retries to resolve uncertainty autonomously. Request assistance for a concrete blocker requiring human intervention; stop for actual unsafe motion or a control anomaly. Final operator evaluation and scene-reset confirmation remain required.",
+ DUAL_FRANKA_TASKS[4].constraints[21],
),
)
diff --git a/robots/dual_franka/toolkit.py b/robots/dual_franka/toolkit.py
index c4c06d2ba..f127f81b9 100644
--- a/robots/dual_franka/toolkit.py
+++ b/robots/dual_franka/toolkit.py
@@ -83,11 +83,12 @@ def __init__(
if attempts_per_session < 0:
raise ValueError("attempts_per_session must be nonnegative")
self._mode = mode
+ self._attended = mode == "exploration" or operator_input is not None
self._attempt = 0 # first attempt begins only after scene confirmation
- self._budget = attempts_per_session
+ self._budget = attempts_per_session if mode == "exploration" else 0
self._attempts_per_session = attempts_per_session
self._session_attempt = 1
- self._scene_ready = mode != "exploration"
+ self._scene_ready = not self._attended
self._operator_input = operator_input
self._operator_verdict = None
self._operator_notes = ""
@@ -97,6 +98,8 @@ def __init__(
self._events = []
self._direct_verdict_event = threading.Event()
self._direct_verdict: str | None = None
+ self._direct_verdict_notes = ""
+ self._session_closed = threading.Event()
super().__init__(
runtime_kwargs=runtime_kwargs,
dashboard_events=dashboard_events,
@@ -109,33 +112,44 @@ def direct_verdict_requested(self) -> bool:
"""Whether operator control has sealed the active session."""
return self._direct_verdict_event.is_set()
- def request_direct_verdict(self, verdict: str) -> bool:
+ def request_direct_verdict(self, verdict: str, notes: str = "") -> bool:
"""Seal this attempt immediately; cancel in-flight work at its next boundary."""
if verdict not in {"success", "failure", "abort"}:
raise ValueError("verdict must be success, failure or abort")
with self._operation_lock:
+ if self._session_closed.is_set():
+ return False
if self._direct_verdict_event.is_set():
- return self._direct_verdict == verdict
- if self._mode != "exploration" or (
+ return False
+ if not self._attended or (
verdict == "success"
and (not self._scene_ready or self._operator_aborted)
):
return False
self._direct_verdict = verdict
+ self._direct_verdict_notes = notes
self._direct_verdict_event.set()
if self._active_operation is not None:
self._active_operation.cancel_event.set()
return True
def raise_if_cancelled(self) -> None:
+ if self._session_closed.is_set():
+ raise ToolCancelled("exploration session closed")
if self._direct_verdict_event.is_set():
raise ToolCancelled(
"operator submitted a terminal verdict; stopping exploration"
)
super().raise_if_cancelled()
- def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult:
- if self._direct_verdict_event.is_set():
+ def execute_tool(
+ self,
+ name: str,
+ input_dict: dict[str, Any],
+ *,
+ cancel_event: threading.Event | None = None,
+ ) -> ToolResult:
+ if self._direct_verdict_event.is_set() or self._session_closed.is_set():
return ToolResult(
data={
"error": (
@@ -144,7 +158,13 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult:
"motion_refused": True,
}
)
- return super().execute_tool(name, input_dict)
+ return super().execute_tool(name, input_dict, cancel_event=cancel_event)
+
+ def close(self) -> None:
+ """Stop and join old tools before their operator input can be reused."""
+ self._session_closed.set()
+ self.cancel_active_and_wait()
+ super().close()
def finalize_direct_verdict(self) -> dict[str, Any]:
"""Record fresh operator evidence after all active work has stopped."""
@@ -161,21 +181,20 @@ def finalize_direct_verdict(self) -> dict[str, Any]:
"operator_finished": True,
"operator_aborted": True,
"verdict_source": "interactive_command",
+ "operator_notes": self._direct_verdict_notes,
}
self._event("verdict", verdict="abort", source="interactive_command")
self._event("finish", **result)
return result
- self.get_env_state(
- command={"action": "observe_for_verdict"}, result={}, elapsed_s=0.0
- )
- self._validate_observation()
+ self._observe_current("observe_for_verdict")
record = self.state.latest_record()
- self._publish_step(record)
self._scene_ready = True
self._operator_verdict = self._direct_verdict
self._operator_notes = (
f"Operator entered /{self._direct_verdict} in the interactive terminal."
)
+ if self._direct_verdict_notes:
+ self._operator_notes += " " + self._direct_verdict_notes
self._verdict_step = record.step_idx
self._event(
"verdict",
@@ -186,9 +205,10 @@ def finalize_direct_verdict(self) -> dict[str, Any]:
result = {
"status": self._direct_verdict,
"operator_verdict": self._direct_verdict,
- "operator_finished": True,
+ "operator_finished": False,
"verdict_source": "interactive_command",
"operator_aborted": False,
+ "operator_notes": self._direct_verdict_notes,
}
self._event("finish", **result)
return result
@@ -225,15 +245,33 @@ def _guard_motion(self, inner, **kwargs) -> ToolResult:
self._clear_verdict()
return inner(**kwargs)
+ def _view_env_state(self, step: int = -1) -> ToolResult:
+ if step >= 0:
+ return dual_franka_tools.view_env_state(step, state=self._state)
+ return self._observe_current("observe_current")
+
+ def _observe_current(self, action: str) -> ToolResult:
+ output = self.get_env_state(
+ command={"action": action}, result={}, elapsed_s=0.0
+ )
+ self._validate_observation()
+ self._publish_step(self.state.latest_record())
+ return output
+
def _current_perception(self, inner, **kwargs) -> ToolResult:
step = kwargs.get("step")
- if not self._scene_ready or (
- step is not None and step != -1 and step < self._attempt_start_step
- ):
+ if not self._scene_ready:
+ return ToolResult(
+ data={
+ "error": "localization refused; use fresh observations after confirmed scene reset"
+ }
+ )
+ if step is not None and step != -1 and step < self.state.latest_step:
return ToolResult(
data={
"error": (
- "localization refused; use fresh observations after confirmed scene reset"
+ f"localization refused: step {step} is stale; use current step "
+ f"{self.state.latest_step} and reselect the target on its image"
)
}
)
@@ -307,10 +345,11 @@ def _request_scene_reset(
kind="reset",
)
self._event("reset_response", response=response)
- if response is None or response.strip().lower() != "done":
- self._operator_aborted = (
- response is None or response.strip().lower() == "abort"
- )
+ response_parts = (response or "").strip().split(maxsplit=1)
+ confirmation = response_parts[0].lower() if response_parts else ""
+ notes = response_parts[1] if len(response_parts) > 1 else ""
+ if confirmation != "done":
+ self._operator_aborted = response is None or confirmation == "abort"
return ToolResult(
data={
"error": "scene reset not confirmed",
@@ -333,6 +372,7 @@ def _request_scene_reset(
"ok": True,
"robot_reset": result,
"scene_reset_confirmed": True,
+ "operator_notes": notes,
"notice": "Scene restored by operator; robot posture reset. Re-localize from the new images.",
}
)
@@ -360,12 +400,8 @@ def _request_operator_verdict(
data={"error": "verdict refused; no active confirmed attempt"}
)
# Save the evidence being judged using the existing camera/state logger.
- self.get_env_state(
- command={"action": "observe_for_verdict"}, result={}, elapsed_s=0.0
- )
- self._validate_observation()
+ self._observe_current("observe_for_verdict")
record = self.state.latest_record()
- self._publish_step(record)
response = self._ask_operator(
f"{question}\nAttempt {self._attempt}, observation step {record.step_idx}. "
"Reply success, failure, continue, or abort; optional notes may follow.",
@@ -382,7 +418,19 @@ def _request_operator_verdict(
self._operator_verdict = verdict
self._operator_notes = notes
self._verdict_step = record.step_idx
- elif verdict != "continue":
+ elif verdict == "continue":
+ observation = self._view_env_state()
+ observation.data.update(
+ ok=True,
+ status=verdict,
+ operator_notes=notes,
+ attempt=self._attempt,
+ evidence_step=record.step_idx,
+ observation_step=self.state.latest_step,
+ operator_aborted=False,
+ )
+ return observation
+ else:
return ToolResult(
data={"error": "invalid operator verdict", "evidence": event}
)
@@ -423,8 +471,8 @@ def _guarded_finish(self, inner, **kwargs) -> ToolResult:
self._event("finish", **result.to_dict())
return result
- def get_env_state(self, *, command, result, elapsed_s):
- if self._mode != "exploration":
+ def get_env_state(self, *, command, result, elapsed_s) -> ToolResult:
+ if not self._attended:
return super().get_env_state(
command=command, result=result, elapsed_s=elapsed_s
)
@@ -447,8 +495,11 @@ def get_env_state(self, *, command, result, elapsed_s):
"operator_verdict": self._operator_verdict,
"attempt_start_step": self._attempt_start_step,
}
- self.state.save("exploration.json", status)
- output.data["exploration"] = status
+ lifecycle = (
+ "exploration" if self._mode == "exploration" else "operator_lifecycle"
+ )
+ self.state.save(f"{lifecycle}.json", status)
+ output.data[lifecycle] = status
if result.get("error"):
output.data["error"] = result["error"]
return output
@@ -483,7 +534,7 @@ def _validate_observation(self) -> None:
def solved(self) -> bool:
if self.flash_options is not None:
return self._flash_solved
- if self._mode != "exploration":
+ if not self._attended:
return self._operator_verdict == "success"
return (
self._scene_ready
@@ -559,6 +610,7 @@ def _register_tools(self) -> None:
}
state_handlers.update(
{
+ "view_env_state": self._view_env_state,
"view_camera_meta": partial(
franka_tools.view_camera_meta, state=self._state
),
@@ -569,16 +621,16 @@ def _register_tools(self) -> None:
),
"request_scene_reset": self._request_scene_reset,
"request_operator_verdict": self._request_operator_verdict
- if self._mode == "exploration"
+ if self._attended
else self._evaluation_verdict,
}
)
for definition in self.declared_tools():
name = definition.name
- if name in _EXPLORATION_ONLY_TOOLS and self._mode != "exploration":
+ if name in _EXPLORATION_ONLY_TOOLS and not self._attended:
continue
handler = state_handlers.get(name) or getattr(self._primitives, name)
- if self._mode == "exploration":
+ if self._attended:
if name in _MOTION_TOOLS:
handler = partial(self._guard_motion, handler)
elif name in {"back_project", "segment"}:
@@ -587,11 +639,7 @@ def _register_tools(self) -> None:
handler = partial(self._describe_exploration_setup, handler)
self.add_tool(definition.with_handler(handler))
finish = self._tools["finish"]
- guard = (
- self._guarded_finish
- if self._mode == "exploration"
- else self._evaluation_finish
- )
+ guard = self._guarded_finish if self._attended else self._evaluation_finish
self.add_tool(finish.with_handler(partial(guard, finish)), replace=True)
def _read_operator_line(self, prompt: str) -> str | None:
@@ -632,15 +680,15 @@ def _evaluation_verdict(
}
)
if verdict == "continue":
- return ToolResult(
- data={
- "ok": True,
- "status": "continue",
- "operator_notes": notes,
- "attempt": self._attempt,
- "notice": "Operator requested more action; do not finish yet.",
- }
+ observation = self._view_env_state()
+ observation.data.update(
+ ok=True,
+ status="continue",
+ operator_notes=notes,
+ attempt=self._attempt,
+ notice="Operator requested more action; do not finish yet.",
)
+ return observation
self._operator_verdict = verdict
self._operator_notes = notes
return ToolResult(
diff --git a/robots/dual_franka/tools.py b/robots/dual_franka/tools.py
index a24e6a450..649a841e1 100644
--- a/robots/dual_franka/tools.py
+++ b/robots/dual_franka/tools.py
@@ -85,7 +85,7 @@ def move_delta(
arm: Annotated[Literal["left", "right"], BeforeValidator(coerce_arm)],
delta_xyz: Annotated[list[float], Field(min_length=3, max_length=3)],
) -> ToolResult:
- """Move one Franka TCP by a bounded world-frame xyz delta in meters.
+ """Move one Franka TCP by a bounded world-frame xyz delta in meters. Completion indicates execution ended; target_reached and final_error_m report accuracy, not a mandatory retry condition.
Args:
arm: Which arm to command; the other arm is left uncommanded.
@@ -102,7 +102,7 @@ def rotate_delta(
arm: Annotated[Literal["left", "right"], BeforeValidator(coerce_arm)],
delta_rpy: Annotated[list[float], Field(min_length=3, max_length=3)],
) -> ToolResult:
- """Rotate one Franka TCP by a bounded world-frame rpy delta in radians.
+ """Rotate one Franka TCP by a bounded world-frame rpy delta in radians. Completion indicates execution ended; target_reached and residual errors report accuracy, not a mandatory retry condition. Protective aborts remain failures.
Args:
arm: Which arm to command; the other arm is left uncommanded.
@@ -147,7 +147,7 @@ def recover_joint_posture(
bool, Field(json_schema_extra={"default": True})
] = True,
) -> ToolResult:
- """Reset both arms to their healthy configured joint posture while preserving each gripper's open/closed state. Closed grippers are re-commanded before/after the joint reset so held objects stay clamped, then both TCPs return near their prior poses."""
+ """Reset both arms to their configured joint posture while preserving each gripper's open/closed state. Closed grippers are re-commanded before/after the joint reset so held objects stay clamped, then both TCPs return near their prior poses. This is an available recovery option, not a mandatory response to an advisory joint-health score."""
self._check_cancelled()
data = self.env.recover_joint_posture(
reason=str(reason), return_to_start=bool(return_to_start)
@@ -196,7 +196,7 @@ def describe_dual_franka_setup(self) -> ToolResult:
"available_primitives": [
tool.name for tool in DualFrankaToolkit.declared_tools()
],
- "operator_guidance": "Named VLA semantic boundaries are segment boundaries, not proof of physical success. Verify images, gripper widths/open flags, joint_health, and projection evidence after every action. recover_joint_posture re-commands and preserves each gripper's open/closed state; inspect its gripper_preserved result before continuing.",
+ "operator_guidance": "Named VLA semantic boundaries are segment boundaries, not proof of physical success. Verify images, gripper widths/open flags, and projection evidence after every action.",
}
)
@@ -471,7 +471,7 @@ def vla_left_place(
int, Field(ge=1, le=20, json_schema_extra={"default": 20})
] = 20,
) -> ToolResult:
- """Run the learned left-placement VLA segment. The active task decides the destination; this tool only defines the capability boundary: left gripper opens and the left TCP lifts.
+ """Run the learned left-placement VLA segment under the configured fixed policy instruction. The prompt argument records planner intent; it does NOT select a destination or change policy conditioning. Use only when the policy's actual placement capability matches the intended destination. Boundary: left gripper opens and the left TCP lifts; this does not prove correct placement.
Args:
prompt: Planner-facing segment intent. This is recorded in the tool result; the current live clean-desk checkpoint still receives its fixed training instruction during policy inference.
@@ -580,7 +580,7 @@ def view_env_state(
*,
state: EnvState,
) -> ToolResult:
- """Read a dual-Franka state snapshot. Configured inline camera views are returned directly; other available views are returned as artifact paths; use read_image to inspect these artifacts."""
+ """Capture a fresh dual-Franka observation when step=-1 (default); a nonnegative step reads that historical snapshot without acquisition. Use the returned step for segmentation and back-projection; do not reuse pixels or masks from an older step after refreshing. Configured inline camera views are returned directly; other available views are returned as artifact paths for targeted read_image inspection."""
images: list[bytes] = []
record = state.get(step)
output = record.to_blob()
diff --git a/robots/franka/env_server.py b/robots/franka/env_server.py
index 36f4014e7..f5f436b3a 100644
--- a/robots/franka/env_server.py
+++ b/robots/franka/env_server.py
@@ -73,7 +73,13 @@ def __getattr__(self, name: str):
remote_method = getattr(self.worker, name)
def invoke(*args, **kwargs):
- return remote_method(*args, **kwargs).wait()[0]
+ results = remote_method(*args, **kwargs).wait()
+ if results is None or len(results) != 1:
+ raise RuntimeError(
+ f"RLinf worker failed during {name}; inspect env/Ray logs. "
+ "The operation's hardware outcome is unknown; do not retry motion automatically."
+ )
+ return results[0]
return invoke
diff --git a/rpent/cli/dashboard.py b/rpent/cli/dashboard.py
index 072e256ff..48dad7353 100644
--- a/rpent/cli/dashboard.py
+++ b/rpent/cli/dashboard.py
@@ -36,6 +36,7 @@
from rpent.memory import MemoryManager
from rpent.planner.base import build_planner
from rpent.robots import get_toolkit
+from rpent.tools.human_in_the_loop import record_operator_outcome
from rpent.utils.config import get_memory_dir
from rpent.utils.logging import get_logger, init_output_dir
@@ -54,10 +55,17 @@ def run_dashboard_session(
parser: argparse.ArgumentParser,
) -> int:
"""Run one long-lived Dashboard Session with sequential fresh TaskRuns."""
- if robot_spec.is_real_robot and not (robot_spec.dashboard or {}).get(
- "external_env", False
+ if (
+ robot_spec.is_real_robot
+ and not (robot_spec.dashboard or {}).get("external_env", False)
+ and (
+ not robot_spec.supports_human_interactive_exploration
+ or args.planner != "codex"
+ )
):
- parser.error("This robot requires operator confirmation in a plain terminal.")
+ parser.error(
+ "Physical Dashboard control requires an attended Codex robot runtime."
+ )
from rpent.dashboard.server import DashboardServer
from rpent.dashboard.session import DashboardSessionController
from rpent.dashboard.state import DashboardState
@@ -83,8 +91,12 @@ def run_dashboard_session(
if component["scope"] == "unique"
}
- if getattr(args, "env_endpoint", None) is not None and not dashboard_spec.get(
- "external_env", False
+ if getattr(args, "env_endpoint", None) is not None and not (
+ dashboard_spec.get("external_env", False)
+ or (
+ robot_spec.is_real_robot
+ and robot_spec.supports_human_interactive_exploration
+ )
):
parser.error(
"Dashboard task control cannot use --env-endpoint because each "
@@ -153,6 +165,17 @@ def run_dashboard_session(
session_root=session_root,
),
)
+ from rpent.dashboard.runtime_health import RuntimeHealthMonitor
+
+ health = RuntimeHealthMonitor(
+ state,
+ {
+ component["name"]: vars(args)[f"{component['name']}_endpoint"]
+ for component in dashboard_spec["runtime_components"]
+ if vars(args).get(f"{component['name']}_endpoint")
+ },
+ )
+ health.start()
try:
controller.run()
if state.session_state == "fatal":
@@ -164,6 +187,8 @@ def run_dashboard_session(
threading.Event().wait()
except KeyboardInterrupt:
state.request_shutdown()
+ finally:
+ health.close()
return 0
@@ -178,6 +203,9 @@ def _run_dashboard_task(
session_root: Path,
) -> str | None:
"""Execute one fresh Dashboard TaskRun against Session-owned services."""
+ human_interactive_run = (
+ robot_spec.is_real_robot and robot_spec.supports_human_interactive_exploration
+ )
task_args = copy.copy(args)
for name, value in claimed.request.items():
setattr(task_args, name, value)
@@ -215,6 +243,13 @@ def _run_dashboard_task(
}
prompt_vars = {**run_config.prompt_vars, "output_dir": output_dir}
session_message = robot_spec.prompts.render("user", variables=prompt_vars)
+ if human_interactive_run:
+ session_message += (
+ "\nDASHBOARD OPERATOR: Before the first motion call request_scene_reset "
+ "and wait for the human's web confirmation. Final success/failure "
+ "requires request_operator_verdict or an explicit operator verdict. "
+ "There is no terminal input; do not ask for shell/TTY replies."
+ )
sessions = max(
1,
int(getattr(task_args, "explore_sessions", 1) or 1),
@@ -255,6 +290,14 @@ def _run_dashboard_task(
video_path=state_output_dir / "episode.mp4",
)
if robot_spec.supports_exploration:
+ operator_kwargs = {}
+ if human_interactive_run:
+ from rpent.dashboard.operator import DashboardOperator
+
+ state.operator = DashboardOperator(
+ state_output_dir, on_change=state.operator_changed
+ )
+ operator_kwargs["operator_input"] = state.operator
toolkit = get_toolkit(
args.robot_name,
runtime_kwargs=runtime_kwargs,
@@ -265,6 +308,7 @@ def _run_dashboard_task(
task_args, "explore_attempts_per_session", 0
),
state_output_dir=state_output_dir,
+ **operator_kwargs,
)
else:
toolkit = get_toolkit(
@@ -303,6 +347,22 @@ def _run_dashboard_task(
messages += result.messages
stats = result.stats
agent_error = result.error
+ if human_interactive_run and toolkit.direct_verdict_requested:
+ finish_result = toolkit.finalize_direct_verdict()
+ if task_args.explore and finish_result["operator_verdict"] in {
+ "success",
+ "failure",
+ }:
+ record_operator_outcome(
+ memory_root=memory_manager.root,
+ recipe_tag=recipe_tag,
+ run_name=output_dir.name,
+ session_number=session_number,
+ state_output_dir=state_output_dir,
+ verdict=finish_result,
+ messages=_serialize_messages(result.messages),
+ planner_error=agent_error,
+ )
if robot_spec.supports_exploration:
solved = bool(toolkit.solved())
if solved:
@@ -375,8 +435,12 @@ def _run_dashboard_task(
getattr(task_args, "explore", False)
and getattr(task_args, "auto_merge_memory", False)
and not agent_error
- and not state.task_replacement_requested
+ and (
+ not state.task_replacement_requested
+ or (human_interactive_run and state.operator_completed_task)
+ )
and memory_manager is not None
+ and (not human_interactive_run or solved)
):
try:
merge_result = memory_manager.merge_memory(
@@ -420,4 +484,11 @@ def _run_dashboard_task(
logger.warning("%s", finalization_error)
else:
agent_error = finalization_error
+ if (
+ human_interactive_run
+ and not solved
+ and not agent_error
+ and not state.task_replacement_requested
+ ):
+ return "Task ended without operator-confirmed success"
return agent_error
diff --git a/rpent/cli/main.py b/rpent/cli/main.py
index cb53ec75a..395fb8e24 100644
--- a/rpent/cli/main.py
+++ b/rpent/cli/main.py
@@ -45,6 +45,8 @@
from pathlib import Path
from rpent.cli.tui import (
+ SessionInputQueue,
+ next_user_line,
start_first_prompt_resolver,
start_interactive_reader,
)
@@ -54,14 +56,153 @@
)
from rpent.evaluation import RunFinalizationContext
from rpent.memory import MemoryManager
-from rpent.planner.base import REASONING_EFFORTS, build_planner
+from rpent.planner.base import (
+ CODEX_REASONING_EFFORTS,
+ REASONING_EFFORTS,
+ PlannerResult,
+ build_planner,
+)
from rpent.planner.check import BASE_URL_ENV_BY_PLANNER
from rpent.robots import enumerate_robots, get_robot_spec, get_toolkit
+from rpent.tools.human_in_the_loop import record_operator_outcome
from rpent.utils.logging import get_logger, init_output_dir
logger = get_logger("agent")
+def _solve_attended(
+ planner, *, operator_input, state_output_dir, keep_mcp_alive=False, **kwargs
+):
+ """Own the transport for the whole attended attempt, including human waits."""
+ if not keep_mcp_alive:
+ return _solve_attended_turns(
+ planner,
+ operator_input=operator_input,
+ state_output_dir=state_output_dir,
+ **kwargs,
+ )
+ from rpent.planner.utils.http_mcp_server import HttpMcpServer
+
+ toolkit = kwargs["toolkit"]
+ server = HttpMcpServer(toolkit)
+ try:
+ server.start()
+ return _solve_attended_turns(
+ planner,
+ operator_input=operator_input,
+ state_output_dir=state_output_dir,
+ mcp_server=server,
+ **kwargs,
+ )
+ finally:
+ # Release pending tool handlers before asking HTTP to shut down.
+ toolkit.cancel_active_and_wait()
+ server.stop()
+
+
+def _solve_attended_turns(planner, *, operator_input, state_output_dir, **kwargs):
+ """Keep an unjudged attempt alive across planner replies and human pauses."""
+ toolkit = kwargs["toolkit"]
+ original_message = kwargs["user_message"]
+ messages = []
+ usage = {}
+ turn_number = 0
+ while True:
+ continue_revision = operator_input.continue_revision
+ result = planner.solve(**kwargs)
+ turn_number += 1
+ messages.extend(result.messages)
+ for key in (
+ "total_input_tokens",
+ "total_output_tokens",
+ "total_cached_input_tokens",
+ "tool_calls",
+ ):
+ usage[key] = usage.get(key, 0) + result.stats.get(key, 0)
+ # Preserve every reply before the planner reuses its output filenames.
+ archive = Path(state_output_dir) / f"dialogue_{turn_number:03d}.json"
+ archive.parent.mkdir(parents=True, exist_ok=True)
+ archive.write_text(
+ json.dumps(
+ {
+ "messages": _serialize_messages(result.messages),
+ "stats": result.stats,
+ "finish": result.finish_result,
+ "error": result.error,
+ },
+ indent=2,
+ default=str,
+ )
+ )
+ if result.error or result.finish_result or toolkit.direct_verdict_requested:
+ return PlannerResult(
+ finish_result=result.finish_result,
+ messages=messages,
+ stats={**result.stats, **usage},
+ error=result.error,
+ )
+ if operator_input.closed:
+ toolkit.request_direct_verdict("abort", "Interactive input closed")
+ return PlannerResult(messages=messages, stats=usage)
+ resumed_operator_request = operator_input.pending_kind is not None
+ if resumed_operator_request:
+ logger.info(
+ "Waiting for the current operator request; keeping this attempt alive."
+ )
+ while (
+ operator_input.pending_kind is not None
+ and not toolkit.direct_verdict_requested
+ and not operator_input.closed
+ ):
+ time.sleep(0.1)
+ toolkit.wait_active()
+ reply = "The pending operator request has returned. Read its recorded result and continue accordingly."
+ else:
+ toolkit.wait_active()
+ if operator_input.continue_revision != continue_revision:
+ resumed_operator_request = True
+ logger.info(
+ "Operator already answered continue; resuming without another input wait."
+ )
+ reply = (
+ "The operator has already answered continue during the preceding turn. "
+ "Do not wait for that same reply again. Read the recorded operator result "
+ "and continue this attempt; ask again only for a new concrete blocker "
+ "or final evaluation."
+ )
+ else:
+ logger.info(
+ "Waiting for your reply in this attempt. Type feedback or /continue; /success, /failure and /abort remain available. No automatic reset."
+ )
+ reply = next_user_line(kwargs["input_queue"])
+ if reply is None and not toolkit.direct_verdict_requested:
+ toolkit.request_direct_verdict("abort", "Interactive input closed")
+ if toolkit.direct_verdict_requested:
+ return PlannerResult(messages=messages, stats=usage)
+ if operator_input.closed:
+ toolkit.request_direct_verdict("abort", "Interactive input closed")
+ return PlannerResult(messages=messages, stats=usage)
+ if not resumed_operator_request:
+ observation = toolkit.execute_tool("view_env_state", {}).data
+ if observation.get("error"):
+ return PlannerResult(
+ messages=messages,
+ stats=usage,
+ error=f"Observation refresh failed after feedback: {observation['error']}",
+ )
+ messages.append({"role": "user", "content": reply})
+ kwargs["user_message"] = (
+ original_message
+ + "\nContinue the SAME attempt, not a new exploration session. Robot state and "
+ "scene confirmation are retained; do not reset merely because the conversation resumed. "
+ f"Current attempt state and operator results are recorded under {state_output_dir}. "
+ "If the operator changed the scene, obtain fresh observations and localization. "
+ "Previous conversation:\n"
+ + json.dumps(_serialize_messages(messages), ensure_ascii=False, default=str)
+ + operator_input.feedback_prompt()
+ )
+
+
# ---------------------------------------------------------------------------
# API agent transcript serialization
# ---------------------------------------------------------------------------
@@ -154,11 +295,12 @@ def _build_argparser() -> argparse.ArgumentParser:
ap.add_argument("--max-tokens", type=int, default=8192)
ap.add_argument(
"--reasoning-effort",
- choices=REASONING_EFFORTS,
+ choices=CODEX_REASONING_EFFORTS,
default="none",
help="Planner reasoning effort for api, claude_code, and "
"codex. Higher effort may improve task success rate "
- "but increases runtime. Defaults to none.",
+ "but increases runtime. Ultra is Codex-only and requires a supporting "
+ "model and SDK. Defaults to none.",
)
ap.add_argument(
"--no-images",
@@ -175,7 +317,7 @@ def _build_argparser() -> argparse.ArgumentParser:
help="Wall-clock cap for api/claude_code/codex planner runs. "
"Terminal interactive API/Claude sessions are exempt. "
"Defaults to CODEX_TIMEOUT_S (codex only), "
- "CELL_TIMEOUT_S, or 1200.",
+ "CELL_TIMEOUT_S, or 1200. For codex, 0 disables this deadline.",
)
ap.add_argument(
"--claude-code-max-budget-usd",
@@ -356,16 +498,23 @@ def main() -> int:
)
args = parser.parse_args()
args.robot_name = early.robot_name
+ if args.planner != "codex" and args.reasoning_effort not in REASONING_EFFORTS:
+ parser.error(
+ f"--reasoning-effort {args.reasoning_effort} requires --planner codex"
+ )
human_interactive_exploration = (
args.explore and robot_spec.supports_human_interactive_exploration
)
+ human_interactive_run = human_interactive_exploration or (
+ args.dashboard and robot_spec.supports_human_interactive_exploration
+ )
if args.dashboard and args.interactive:
parser.error("--dashboard and --interactive cannot be used together")
external_env_dashboard = (getattr(robot_spec, "dashboard", None) or {}).get(
"external_env", False
)
if robot_spec.is_real_robot and not (
- human_interactive_exploration or external_env_dashboard
+ human_interactive_run or external_env_dashboard
):
if args.dashboard or args.interactive:
parser.error(
@@ -398,11 +547,7 @@ def main() -> int:
)
if args.explore and getattr(args, "explore_attempts_per_session", 0) < 0:
parser.error("--explore-attempts-per-session must be nonnegative")
- if human_interactive_exploration:
- if args.dashboard:
- parser.error(
- "Human-interactive exploration currently requires the CLI operator terminal; Dashboard feedback is not implemented"
- )
+ if human_interactive_run and not args.dashboard:
if sys.stdin is None or not sys.stdin.isatty():
parser.error(
"Human-interactive exploration requires a TTY for operator reset/verdict feedback"
@@ -473,12 +618,12 @@ def main() -> int:
)
operator_input = None
- if human_interactive_exploration:
+ if human_interactive_run:
from rpent.tools.human_in_the_loop import HumanInTheLoopInput
- # The native CLI reads between runs, leaving the TTY available to tools.
operator_input = HumanInTheLoopInput(
- interactive=args.interactive and not native_cli
+ interactive=args.interactive and not native_cli,
+ feedback_path=Path(output_dir) / "operator_feedback.jsonl",
)
input_queue: "queue.Queue[str | None] | None" = None
await_first_prompt: "Callable[[], str | None] | None" = None
@@ -527,7 +672,7 @@ def main() -> int:
first_user_msg = await_first_prompt()
if first_user_msg is None:
logger.info("no task entered; ending session before start.")
- if human_interactive_exploration:
+ if human_interactive_run:
prompt_vars = {**prompt_vars, "initial_user_message": first_user_msg}
# Exploration may hand off between independent planner contexts.
sessions = max(1, int(getattr(args, "explore_sessions", 1) or 1))
@@ -538,11 +683,15 @@ def main() -> int:
environment_success: bool | None = None
memory_manager: MemoryManager | None = None
direct_operator_success = False
+ verdict_handoff = ""
try:
if first_user_msg is not None:
dashboard_events.emit(RunStartedEvent())
session_msg = first_user_msg
for session_number in range(1, sessions + 1):
+ direct_operator_success = False
+ continue_after_verdict = False
+ solved = False
if session_msg is None:
break
if session_number > 1:
@@ -556,6 +705,9 @@ def main() -> int:
session_number=session_number,
session_max=sessions,
)
+ session_msg += verdict_handoff
+ if operator_input is not None and args.interactive:
+ session_msg += operator_input.feedback_prompt()
state_output_dir = output_dir
if getattr(args, "explore", False):
state_output_dir = (
@@ -586,33 +738,53 @@ def main() -> int:
config=run_config,
)
memory_manager = toolkit.memory
+ session_input = input_queue
if operator_input is not None and input_queue is not None:
-
- def accept_verdict(verdict: str, active_toolkit=toolkit) -> bool:
- if not active_toolkit.request_direct_verdict(verdict):
+ session_input = SessionInputQueue(input_queue)
+
+ def accept_verdict(
+ verdict: str,
+ notes: str,
+ active_toolkit=toolkit,
+ active_input=session_input,
+ ) -> bool:
+ if not active_toolkit.request_direct_verdict(verdict, notes):
return False
logger.info(
- "/%s accepted: stopping actions, then recording the result and exiting.",
+ "/%s accepted: stopping this attempt and recording its result.",
verdict,
)
- # EOF is a planner control signal, never a model message.
- input_queue.put(None)
+ active_input.cancel()
return True
operator_input.bind_verdict(accept_verdict)
+ pending_operator_request = None
try:
- result = planner.solve(
+ solve = planner.solve
+ if human_interactive_run and input_queue is not None:
+ from functools import partial
+
+ solve = partial(
+ _solve_attended,
+ planner,
+ keep_mcp_alive=args.planner == "codex",
+ operator_input=operator_input,
+ state_output_dir=state_output_dir,
+ )
+ result = solve(
system_prompt=system_prompt,
user_message=session_msg,
toolkit=toolkit,
max_turns=args.max_turns,
- input_queue=input_queue,
+ input_queue=session_input,
)
finish_result = result.finish_result
messages += result.messages
stats = result.stats
agent_error = result.error
- if getattr(toolkit, "direct_verdict_requested", False):
+ if operator_input is not None and args.interactive:
+ pending_operator_request = operator_input.pending_kind
+ if human_interactive_run and toolkit.direct_verdict_requested:
finish_result = toolkit.finalize_direct_verdict()
direct_operator_success = finish_result["status"] == "success"
if agent_error:
@@ -620,17 +792,66 @@ def accept_verdict(verdict: str, active_toolkit=toolkit) -> bool:
"Planner stopped after operator verdict: %s", agent_error
)
stats["planner_error_at_operator_verdict"] = agent_error
- solved_fn = getattr(toolkit, "solved", None)
- if getattr(robot_spec, "supports_exploration", False) and callable(
- solved_fn
- ):
- solved = bool(solved_fn())
- write_recipe = getattr(toolkit, "write_recipe", None)
- if solved and callable(write_recipe):
- recipe_path = write_recipe(recipe_tag) or recipe_path
+ verdict = finish_result["operator_verdict"]
+ if (
+ verdict in {"success", "failure"}
+ and human_interactive_exploration
+ ):
+ evidence_path = record_operator_outcome(
+ memory_root=memory_manager.root,
+ recipe_tag=recipe_tag,
+ run_name=Path(output_dir).name,
+ session_number=session_number,
+ state_output_dir=state_output_dir,
+ verdict=finish_result,
+ messages=_serialize_messages(result.messages),
+ planner_error=agent_error,
+ )
+ verdict_handoff = (
+ f"\nThe operator marked the preceding attempt as {verdict}. "
+ f"Before requesting any reset or motion, read {evidence_path}, "
+ "inspect its evidence and write a grounded outcome summary "
+ "to your memory inbox wip notes. Distinguish observations "
+ "from hypotheses; never label a failed attempt as successful. "
+ "Then request_scene_reset and wait for a new operator "
+ "confirmation before attempting the task again.\n"
+ )
+ # An operator verdict deliberately interrupts the planner.
+ # Its cancellation is not a failed continuation session.
+ if verdict == "failure":
+ agent_error = None
+ continue_after_verdict = not agent_error
+ if robot_spec.supports_exploration:
+ solved = bool(toolkit.solved())
+ if solved:
+ recipe_path = toolkit.write_recipe(recipe_tag) or recipe_path
+ if (
+ human_interactive_exploration
+ and direct_operator_success
+ and not agent_error
+ ):
+ # Publish before the next session can overwrite the
+ # run-level recipe; retain a per-session copy as evidence.
+ import shutil
+
+ for name in (
+ f"{recipe_tag}.json",
+ f"{recipe_tag}_recipe.jsonl",
+ ):
+ shutil.copy2(
+ Path(output_dir) / name,
+ Path(state_output_dir) / name,
+ )
+ memory_manager.merge_memory(
+ cell_tag=recipe_tag,
+ run_state_dir=state_output_dir,
+ solved=True,
+ )
finally:
- if operator_input is not None and input_queue is not None:
+ if operator_input is not None and args.interactive:
operator_input.bind_verdict(None)
+ if session_input is not None:
+ session_input.cancel()
try:
if robot_spec.finalize_run is not None:
solved_fn = getattr(toolkit, "solved", None)
@@ -640,8 +861,23 @@ def accept_verdict(verdict: str, active_toolkit=toolkit) -> bool:
solved = bool(environment_success)
finally:
toolkit.close()
+ if operator_input is not None and args.interactive:
+ operator_input.cancel_pending()
+ if (
+ human_interactive_run
+ and not continue_after_verdict
+ and not (finish_result or {}).get("_finish")
+ and not (finish_result or {}).get("operator_finished")
+ ):
+ agent_error = agent_error or (
+ f"Planner returned without an attempt verdict; pending operator request: "
+ f"{pending_operator_request or 'none'}. Old tools were cancelled. "
+ "Stopping exploration instead of starting another empty session."
+ )
+ logger.error("%s", agent_error)
+ break
if (
- solved
+ (solved and not continue_after_verdict)
or (finish_result or {}).get("operator_aborted")
or (finish_result or {}).get("operator_finished")
):
diff --git a/rpent/cli/tui.py b/rpent/cli/tui.py
index 747f5dfa4..be3999fd2 100644
--- a/rpent/cli/tui.py
+++ b/rpent/cli/tui.py
@@ -22,7 +22,9 @@
import queue
import sys
import threading
+import time
from collections.abc import Callable
+from typing import Protocol
from prompt_toolkit import PromptSession
from prompt_toolkit.history import InMemoryHistory
@@ -45,6 +47,67 @@
"""
+class InputQueue(Protocol):
+ """Input operations shared by the terminal reader and planner consumers."""
+
+ def get(self, block: bool = True, timeout: float | None = None) -> str | None:
+ """Read a submitted line or the terminal's EOF signal."""
+ ...
+
+ def put(self, item: str | None) -> None:
+ """Forward a line or the terminal's EOF signal."""
+ ...
+
+
+class SessionInputQueue:
+ """Cancel one input consumer without putting EOF in the shared queue.
+
+ A scope may wrap another scope, allowing a planner reply to end without
+ cancelling its attended attempt or consuming the next attempt's input.
+ """
+
+ def __init__(self, source: InputQueue) -> None:
+ self._source = source
+ self._cancelled = threading.Event()
+ self._lock = threading.Lock()
+
+ @property
+ def cancelled(self) -> bool:
+ """Return whether this scope has been cancelled."""
+ return self._cancelled.is_set()
+
+ def cancel(self) -> None:
+ """Wake this scope's readers without closing or changing its source."""
+ with self._lock:
+ self._cancelled.set()
+
+ def get(self, block: bool = True, timeout: float | None = None) -> str | None:
+ """Read input, returning None when this scope or its source closes."""
+ if block and timeout is not None and timeout < 0:
+ raise ValueError("timeout must be a non-negative number")
+ deadline = time.monotonic() + timeout if timeout is not None else None
+ while True:
+ with self._lock:
+ if self.cancelled:
+ return None
+ try:
+ return self._source.get(block=False)
+ except queue.Empty:
+ if not block:
+ raise
+ wait_time = 0.1
+ if deadline is not None:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise queue.Empty
+ wait_time = min(wait_time, remaining)
+ self._cancelled.wait(wait_time)
+
+ def put(self, item: str | None) -> None:
+ """Forward real input to the source, including returned unread lines."""
+ self._source.put(item)
+
+
def handle_local_command(line: str, *, extra_help: str = "") -> bool:
"""Handle TUI-local commands; return True when the line was consumed."""
if line.strip().lower() not in HELP_TOKENS:
@@ -149,7 +212,7 @@ def _restore() -> None:
def start_interactive_reader(
- input_queue: "queue.Queue[str | None]",
+ input_queue: InputQueue,
*,
first_prompt_default: str | None = None,
line_handler: Callable[[str], bool] | None = None,
@@ -203,7 +266,7 @@ def _read() -> None:
return thread
-def next_user_line(input_queue: "queue.Queue[str | None]") -> str | None:
+def next_user_line(input_queue: InputQueue) -> str | None:
"""Block for the next actionable user line from an interactive input queue.
Returns the trimmed line, or ``None`` when the session should end (the queue
@@ -223,7 +286,7 @@ def next_user_line(input_queue: "queue.Queue[str | None]") -> str | None:
def initial_user_message(
- input_queue: "queue.Queue[str | None]",
+ input_queue: InputQueue,
) -> str | None:
"""Block for the first user turn of an interactive session.
@@ -243,7 +306,7 @@ def initial_user_message(
def start_first_prompt_resolver(
- input_queue: "queue.Queue[str | None]",
+ input_queue: InputQueue,
) -> Callable[[], str | None]:
"""Resolve the opening user turn on a background thread.
diff --git a/rpent/dashboard/operator.py b/rpent/dashboard/operator.py
new file mode 100644
index 000000000..c76720718
--- /dev/null
+++ b/rpent/dashboard/operator.py
@@ -0,0 +1,98 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Cancelable, request-scoped operator confirmations for physical tasks."""
+
+import json
+import threading
+import uuid
+from pathlib import Path
+
+from rpent.tools.toolkit import ToolCancelled
+
+
+class DashboardOperator:
+ def __init__(self, output_dir, on_change=lambda: None):
+ self._condition = threading.Condition()
+ self._pending = None
+ self._answer = None
+ self._closed = False
+ self._path = Path(output_dir) / "dashboard_operator.jsonl"
+ self._on_change = on_change
+
+ def _record(self, event):
+ self._path.parent.mkdir(parents=True, exist_ok=True)
+ with self._path.open("a") as file:
+ file.write(json.dumps(event, ensure_ascii=False) + "\n")
+
+ def snapshot(self):
+ with self._condition:
+ return dict(self._pending) if self._pending else None
+
+ def request(self, prompt, check_cancelled, *, kind=None):
+ choices = (
+ ["done", "abort"]
+ if kind == "reset"
+ else ["success", "failure", "continue", "abort"]
+ )
+ with self._condition:
+ if self._closed:
+ raise ToolCancelled("operator channel closed")
+ if self._pending:
+ raise RuntimeError("another operator confirmation is pending")
+ self._pending = {
+ "id": uuid.uuid4().hex,
+ "kind": kind,
+ "prompt": prompt,
+ "choices": choices,
+ }
+ self._answer = None
+ self._record({"event": "request", **self._pending})
+ self._on_change()
+ try:
+ with self._condition:
+ while self._answer is None:
+ check_cancelled()
+ if self._closed:
+ raise ToolCancelled("operator channel closed")
+ self._condition.wait(0.1)
+ check_cancelled()
+ return self._answer
+ finally:
+ with self._condition:
+ self._pending = None
+ self._answer = None
+ self._on_change()
+
+ def reply(self, request_id, answer, notes=""):
+ with self._condition:
+ if (
+ self._closed
+ or not self._pending
+ or request_id != self._pending["id"]
+ or self._answer is not None
+ ):
+ raise ValueError("expired or already answered operator request")
+ if answer not in self._pending["choices"] or not isinstance(notes, str):
+ raise ValueError("invalid operator response")
+ self._answer = answer + (" " + notes.strip() if notes.strip() else "")
+ self._record(
+ {"event": "reply", "id": request_id, "answer": answer, "notes": notes}
+ )
+ self._condition.notify_all()
+
+ def close(self):
+ with self._condition:
+ self._closed = True
+ self._condition.notify_all()
diff --git a/rpent/dashboard/runtime_health.py b/rpent/dashboard/runtime_health.py
new file mode 100644
index 000000000..b25bf8b56
--- /dev/null
+++ b/rpent/dashboard/runtime_health.py
@@ -0,0 +1,49 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Read-only liveness checks; never reconnect/reset a robot automatically."""
+
+import threading
+
+from rpent.dashboard.events import RuntimeStatusEvent
+from rpent.utils.rpc import make_rpc_client
+
+
+class RuntimeHealthMonitor:
+ def __init__(self, state, endpoints):
+ self._state = state
+ self._clients = {name: make_rpc_client(url) for name, url in endpoints.items()}
+ self._stop = threading.Event()
+ self._thread = threading.Thread(target=self._run, daemon=True)
+
+ def start(self):
+ self._thread.start()
+
+ def close(self):
+ self._stop.set()
+ self._thread.join(timeout=5)
+
+ def check(self):
+ for name, client in self._clients.items():
+ try:
+ client.call("healthz", timeout_s=1)
+ event = RuntimeStatusEvent(name, "ready")
+ except Exception as exc:
+ event = RuntimeStatusEvent(name, "failed", str(exc))
+ self._state.emit(event)
+
+ def _run(self):
+ while not self._stop.is_set():
+ self.check()
+ self._stop.wait(2)
diff --git a/rpent/dashboard/server.py b/rpent/dashboard/server.py
index 7c9549411..7e67dd006 100644
--- a/rpent/dashboard/server.py
+++ b/rpent/dashboard/server.py
@@ -187,7 +187,10 @@ def api_submit_message(
payload: dict[str, Any] = Body(default={}),
) -> JSONResponse:
try:
- self._state.submit_input(payload.get("text"))
+ self._state.submit_input(
+ payload.get("text"),
+ operator_context=payload.get("operator_context"),
+ )
except ValueError as exc:
return JSONResponse({"error": str(exc)}, status_code=422)
except InteractionUnavailableError as exc:
diff --git a/rpent/dashboard/state.py b/rpent/dashboard/state.py
index 4c26f98b7..3d4148a7c 100644
--- a/rpent/dashboard/state.py
+++ b/rpent/dashboard/state.py
@@ -45,6 +45,7 @@
UnknownDashboardMessageError,
)
from rpent.dashboard.spec import DashboardSpec, TaskSpec
+from rpent.tools.human_in_the_loop import parse_operator_command
from rpent.utils.logging import get_logger
from rpent.utils.templates import substitute
@@ -193,7 +194,7 @@ def __init__(
}
self._primitive_allowlist = dashboard_spec["primitives"]
- self._lock = threading.Lock()
+ self._lock = threading.RLock()
self._condition = threading.Condition(self._lock)
self._next_model: str | None = None
self._task_state: str | None = None
@@ -235,6 +236,54 @@ def __init__(
self._manual_results: list[str] = []
self._explore_paused = False
self._exploration_message: DashboardMessage | None = None
+ self.operator = None
+ self.operator_verdict = None
+
+ def operator_changed(self):
+ with self._condition:
+ self._projection_changed_locked()
+
+ def operator_snapshot(self):
+ with self._condition:
+ return {
+ "enabled": self.operator is not None,
+ "active": self._toolkit is not None,
+ "generation": self._task_generation,
+ "pending": self.operator.snapshot() if self.operator else None,
+ "verdict": self.operator_verdict,
+ }
+
+ def operator_reply(self, generation, request_id, answer, notes=""):
+ with self._condition:
+ if (
+ generation != self._task_generation
+ or self.operator is None
+ or self._session_state == "switch_pending"
+ ):
+ raise ValueError("expired operator task")
+ self.operator.reply(request_id, answer, notes)
+
+ def operator_finish(self, generation, verdict, notes=""):
+ with self._condition:
+ if verdict not in {"success", "failure", "abort"} or not isinstance(
+ notes, str
+ ):
+ raise ValueError("invalid operator verdict")
+ if (
+ generation != self._task_generation
+ or self.operator is None
+ or self._toolkit is None
+ or self._session_state == "switch_pending"
+ ):
+ raise ValueError("no matching active operator task")
+ if not self._toolkit.request_direct_verdict(verdict, notes):
+ raise ValueError("operator verdict refused in current state")
+ self.operator_verdict = verdict
+ self._pending_task = None
+ self._session_state = "switch_pending"
+ self._accepting_input = False
+ self.operator.close()
+ self._interaction_changed_locked()
def bind_toolkit(self, toolkit: Toolkit) -> None:
"""Expose a TaskRun Toolkit through the Dashboard primitive API."""
@@ -250,6 +299,8 @@ def unbind_toolkit(self, toolkit: Toolkit) -> None:
with self._condition:
if self._toolkit is toolkit:
self._toolkit = None
+ if self.operator is not None:
+ self.operator.close()
self._projection_changed_locked()
toolkit.cancel_active_and_wait()
with self._condition:
@@ -394,6 +445,16 @@ def task_replacement_requested(self) -> bool:
with self._lock:
return self._session_state == "switch_pending"
+ @property
+ def operator_completed_task(self) -> bool:
+ """Whether operator completion is draining without a queued replacement."""
+ with self._lock:
+ return (
+ self._session_state == "switch_pending"
+ and self.operator_verdict is not None
+ and self._pending_task is None
+ )
+
def shared_services_ready(self) -> None:
"""Open the command channel after shared services have started."""
with self._condition:
@@ -475,7 +536,9 @@ def continue_exploration(self, *, explicit: bool = False) -> None:
self._active_primitive_calls.pop(key, None)
self._interaction_changed_locked()
- def submit_input(self, text: str) -> DashboardMessage | TaskRequest:
+ def submit_input(
+ self, text: str, *, operator_context=None
+ ) -> DashboardMessage | TaskRequest | None:
"""Route a local task command or a normal conversation message."""
if not isinstance(text, str) or not text.strip():
return self._submit_message(text)
@@ -486,6 +549,38 @@ def submit_input(self, text: str) -> DashboardMessage | TaskRequest:
raise ValueError(guidance)
self.continue_exploration(explicit=True)
return {"command": command}
+ with self._condition:
+ parsed = parse_operator_command(text) if self.operator is not None else None
+ if parsed is not None:
+ command = f"/{parsed.name}"
+ notes = parsed.notes
+ current = self.operator_snapshot()
+ if operator_context is None:
+ raise ValueError(
+ "No active operator channel; refresh the Dashboard."
+ )
+ if not isinstance(operator_context, dict):
+ raise ValueError("operator_context must be an object.")
+ if operator_context.get("generation") != current["generation"]:
+ raise ValueError("expired operator task")
+ pending = current["pending"]
+ if command == "/continue" and pending is None:
+ return self._submit_message(text)
+ if command in {"/success", "/failure", "/abort"}:
+ self.operator_finish(current["generation"], command[1:], notes)
+ else:
+ request_id = operator_context.get("request_id")
+ answer = command[1:]
+ if command == "/operator":
+ request_id, answer = parsed.request_id, parsed.answer
+ if pending is None or request_id != pending["id"]:
+ raise ValueError("No matching pending operator request.")
+ self.operator_reply(
+ current["generation"], request_id, answer, notes
+ )
+ self._control_feedback = [f"{command} accepted"]
+ self._interaction_changed_locked()
+ return None
try:
request = _parse_task(self._task_spec, text)
except ValueError as exc:
@@ -591,6 +686,12 @@ def complete_task(
if state not in TERMINAL_RUN_STATES:
raise ValueError(f"invalid terminal run state: {state!r}")
with self._condition:
+ if self.operator_verdict is not None and error is None:
+ state = {
+ "success": "succeeded",
+ "failure": "failed",
+ "abort": "cancelled",
+ }[self.operator_verdict]
self._task_state = state
self._error = None if error is None else str(error)
self._seal_interaction_locked()
@@ -609,6 +710,8 @@ def _begin_task_locked(
output_dir: Path,
) -> None:
self._current_task = request
+ self.operator = None
+ self.operator_verdict = None
self.video_path = output_dir / "episode.mp4"
self._session_state = "task_starting"
self._task_state = "starting"
@@ -1083,6 +1186,7 @@ def _session_fields_locked(self) -> dict[str, Any]:
"pending_task": self._command_snapshot(self._pending_task),
"control_feedback": list(self._control_feedback),
"control_error": self._control_error,
+ "operator": self.operator_snapshot(),
}
def _visible_state_locked(self) -> str:
diff --git a/rpent/dashboard/static/interaction.js b/rpent/dashboard/static/interaction.js
index 1382d9eec..166cd68a7 100644
--- a/rpent/dashboard/static/interaction.js
+++ b/rpent/dashboard/static/interaction.js
@@ -249,6 +249,10 @@ export function createInteractionController({ copy, select }) {
const backendError = errorText(interaction.last_error);
const controlError = errorText(session.control_error);
const feedback = session.control_feedback.map(errorText).filter(Boolean);
+ const operator = session.operator;
+ if (operator?.enabled) {
+ select("#chatHint").textContent += " · /done · /continue · /success · /failure · /abort";
+ }
if (state.requestError) {
status.textContent = state.requestError;
status.classList.add("is-error");
@@ -267,6 +271,11 @@ export function createInteractionController({ copy, select }) {
} else if (interaction.interrupt_requested || state.interruptInFlight) {
status.textContent = copy.interruptRequested;
status.classList.add("is-busy");
+ } else if (operator?.pending) {
+ status.textContent = operator.pending.prompt + (operator.pending.kind === "reset"
+ ? "\nEnter /done to confirm scene restoration and allow the robot to reset, or /abort to cancel."
+ : "\nEnter /continue, /success, /failure or /abort, optionally followed by notes.");
+ status.classList.add("is-ready");
} else if (state.notice) {
status.textContent = state.notice;
status.classList.add("is-ready");
@@ -380,7 +389,13 @@ export function createInteractionController({ copy, select }) {
try {
await requestJSON(
"/api/session/messages",
- { method: "POST", body: { text } },
+ { method: "POST", body: {
+ text,
+ operator_context: {
+ generation: state.snapshot.task_generation,
+ request_id: state.snapshot.operator?.pending?.id ?? null,
+ },
+ } },
);
if (input.value === draft) input.value = "";
} catch (error) {
diff --git a/rpent/planner/base.py b/rpent/planner/base.py
index b90140572..1bf0224e4 100644
--- a/rpent/planner/base.py
+++ b/rpent/planner/base.py
@@ -17,7 +17,6 @@
from __future__ import annotations
import os
-import queue
from abc import ABC, abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING
@@ -33,10 +32,13 @@
if TYPE_CHECKING:
from pydantic_ai.models import Model
+ from rpent.cli.tui import InputQueue
+
#: MCP namespace prefix for RPent tools (``mcp____``).
#: Toolkits expose plain tool names; planners add/strip this prefix.
MCP_TOOL_PREFIX = "mcp__rpent__"
REASONING_EFFORTS = ("none", "low", "medium", "high", "xhigh")
+CODEX_REASONING_EFFORTS = (*REASONING_EFFORTS, "ultra")
def resolve_model(planner: str, model: str | None) -> str | None:
@@ -96,7 +98,7 @@ def solve(
user_message: str,
toolkit: Toolkit,
max_turns: int,
- input_queue: queue.Queue[str | None] | None = None,
+ input_queue: InputQueue | None = None,
dashboard_interaction: DashboardInteractionPort | None = None,
) -> PlannerResult:
"""Run the multi-turn agent loop until completion or budget.
diff --git a/rpent/planner/claude_code.py b/rpent/planner/claude_code.py
index 9596385fc..644304acd 100644
--- a/rpent/planner/claude_code.py
+++ b/rpent/planner/claude_code.py
@@ -27,7 +27,6 @@
import contextlib
import dataclasses
import json
-import queue
import tempfile
import time
from collections.abc import Callable
@@ -35,7 +34,7 @@
from pathlib import Path
from typing import Any
-from rpent.cli.tui import next_user_line
+from rpent.cli.tui import InputQueue, SessionInputQueue, next_user_line
from rpent.dashboard.events import (
DashboardEventSink,
TranscriptEvent,
@@ -103,7 +102,7 @@ def solve(
user_message: str,
toolkit: Toolkit,
max_turns: int,
- input_queue: queue.Queue[str | None] | None = None,
+ input_queue: InputQueue | None = None,
dashboard_interaction: DashboardInteractionPort | None = None,
) -> PlannerResult:
"""Run a Claude Agent SDK session for the given prompt."""
@@ -132,7 +131,7 @@ async def _solve_async(
initial_user_text: str,
toolkit: Toolkit,
max_turns: int,
- input_queue: queue.Queue[str | None] | None = None,
+ input_queue: InputQueue | None = None,
dashboard_interaction: DashboardInteractionPort | None = None,
) -> PlannerResult:
import claude_agent_sdk
@@ -472,8 +471,8 @@ async def _consume(self, adapter: Any) -> None:
class _TerminalSessionAdapter:
"""Preserve the terminal TUI's interrupt-then-query steering policy."""
- def __init__(self, *, input_queue: Any, emit_user) -> None:
- self._input_queue = input_queue
+ def __init__(self, *, input_queue: InputQueue, emit_user) -> None:
+ self._input_queue = SessionInputQueue(input_queue)
self._emit_user = emit_user
async def initial_query_succeeded(self, driver: _ClaudeSessionDriver) -> None:
@@ -502,8 +501,7 @@ async def on_message(
return None
async def close(self) -> None:
- # Unblock next_user_line() if the consumer (for example finish) won.
- self._input_queue.put(None)
+ self._input_queue.cancel()
class _ClaudeDashboardAdapter:
diff --git a/rpent/planner/codex.py b/rpent/planner/codex.py
index e9f566473..5cd1442e2 100644
--- a/rpent/planner/codex.py
+++ b/rpent/planner/codex.py
@@ -29,7 +29,6 @@
import contextlib
import json
import os
-import queue
import tempfile
import threading
import time
@@ -40,7 +39,7 @@
import openai_codex
from openai_codex.generated.v2_all import ReasoningEffort
-from rpent.cli.tui import next_user_line
+from rpent.cli.tui import InputQueue, SessionInputQueue, next_user_line
from rpent.dashboard.events import (
DashboardEventSink,
TranscriptEvent,
@@ -49,12 +48,12 @@
from rpent.dashboard.interaction import DashboardInteractionPort, DashboardMessage
from rpent.dashboard.planner_control import DashboardPlannerControl
from rpent.planner.base import (
- REASONING_EFFORTS,
+ CODEX_REASONING_EFFORTS,
Planner,
PlannerResult,
strip_mcp_prefix,
)
-from rpent.planner.utils.http_mcp_server import HttpMcpServer
+from rpent.planner.utils.http_mcp_server import HttpMcpServer, mcp_tool_timeout_s
from rpent.tools.toolkit import Toolkit
from rpent.utils.config import get_repo_root
from rpent.utils.logging import get_logger
@@ -104,7 +103,9 @@ def __init__(
"""Initialize the Codex SDK backend."""
self._output_dir = str(output_dir)
self._repo_root = str(repo_root) if repo_root else str(get_repo_root())
- self._timeout_s = timeout_s
+ if timeout_s < 0:
+ raise ValueError("timeout_s must be nonnegative (0 disables the deadline)")
+ self._timeout_s = None if timeout_s == 0 else timeout_s
self._extra_dirs = extra_dirs or []
self._output_path = Path(output_path) if output_path else None
self._model = model or os.environ.get("CODEX_MODEL", None)
@@ -112,7 +113,7 @@ def __init__(
self._api_key = os.environ.get("CODEX_API_KEY", None)
self._service_tier = os.environ.get("CODEX_SERVICE_TIER", None)
self._dashboard_events = dashboard_events
- if reasoning_effort not in REASONING_EFFORTS:
+ if reasoning_effort not in CODEX_REASONING_EFFORTS:
raise ValueError(f"unsupported reasoning effort: {reasoning_effort}")
self._thread_options = {
"approval_mode": openai_codex.ApprovalMode.deny_all,
@@ -134,8 +135,9 @@ def solve(
user_message: str,
toolkit: Toolkit,
max_turns: int,
- input_queue: queue.Queue[str | None] | None = None,
+ input_queue: InputQueue | None = None,
dashboard_interaction: DashboardInteractionPort | None = None,
+ mcp_server: HttpMcpServer | None = None,
) -> PlannerResult:
"""Run one or more Codex SDK turns for the given prompt."""
if input_queue is not None and dashboard_interaction is not None:
@@ -144,6 +146,8 @@ def solve(
)
prompt = f"{system_prompt}\n\n{user_message}" if system_prompt else user_message
if dashboard_interaction is not None:
+ if mcp_server is not None:
+ raise ValueError("dashboard owns its MCP server")
return asyncio.run(
self._solve_dashboard(
prompt=prompt,
@@ -162,7 +166,9 @@ def solve(
# Start the in-thread MCP HTTP server so Codex can reach the
# shared toolkit without spawning a subprocess.
- mcp_server = HttpMcpServer(toolkit)
+ owns_mcp_server = mcp_server is None
+ if owns_mcp_server:
+ mcp_server = HttpMcpServer(toolkit)
mcp_url = mcp_server.start()
logger.info("mcp http endpoint: %s", mcp_url)
@@ -170,9 +176,9 @@ def solve(
logger.info("prompt: %d chars", len(prompt))
logger.info("output_dir: %s", self._output_dir)
logger.info(
- "invoking Codex SDK model %s (timeout=%ds)",
+ "invoking Codex SDK model %s (timeout=%s)",
model_desc,
- self._timeout_s,
+ "unlimited" if self._timeout_s is None else f"{self._timeout_s}s",
)
started = time.time()
@@ -223,7 +229,8 @@ def solve(
_write_jsonl(raw_f, {"type": "error", "message": error})
logger.info(rendered.rstrip())
finally:
- mcp_server.stop()
+ if owns_mcp_server:
+ mcp_server.stop()
elapsed = time.time() - started
text = state.get("text", "") or output_path.read_text(errors="replace")
@@ -275,7 +282,7 @@ def _run_session(
recorder: "_Recorder",
state: dict[str, Any],
mcp_url: str,
- input_queue: "queue.Queue[str | None] | None" = None,
+ input_queue: InputQueue | None = None,
) -> None:
try:
chunks: list[str] = []
@@ -293,20 +300,23 @@ def _run_session(
turn = thread.turn(prompt, **self._turn_options)
state["turn"] = turn
- stop_steer: threading.Event | None = None
+ steering_input: SessionInputQueue | None = None
+ steer_thread: threading.Thread | None = None
if input_queue is not None:
- stop_steer = threading.Event()
+ steering_input = SessionInputQueue(input_queue)
def _steer() -> None:
while True:
- nxt = next_user_line(input_queue)
- if stop_steer.is_set():
- return
+ nxt = next_user_line(steering_input)
if nxt is None:
- try:
- turn.interrupt()
- except Exception:
- pass
+ if not steering_input.cancelled:
+ try:
+ turn.interrupt()
+ except Exception:
+ pass
+ return
+ if steering_input.cancelled:
+ input_queue.put(nxt)
return
rendered = f"\n[user] {nxt}\n"
with write_lock:
@@ -316,6 +326,9 @@ def _steer() -> None:
logger.info(rendered.strip())
try:
turn.steer(nxt)
+ logger.info(
+ "[feedback] delivered to active Codex turn"
+ )
except Exception as e:
rendered = f"\n[codex-planner] steer failed: {e}\n"
with write_lock:
@@ -325,11 +338,12 @@ def _steer() -> None:
logger.info(rendered.strip())
return
- threading.Thread(
+ steer_thread = threading.Thread(
target=_steer,
name="codex-steer",
daemon=True,
- ).start()
+ )
+ steer_thread.start()
limit_reached = False
try:
@@ -355,10 +369,10 @@ def _steer() -> None:
raise
# The terminal notification may still be queued.
finally:
- if stop_steer is not None:
- stop_steer.set()
- if input_queue is not None:
- input_queue.put(None)
+ if steering_input is not None:
+ steering_input.cancel()
+ if steer_thread is not None:
+ steer_thread.join(timeout=2)
state["text"] = "".join(chunks)
if recorder.final_response is not None:
@@ -996,6 +1010,9 @@ def _codex_mcp_config_overrides(
config: list[tuple[str, Any]] = []
if mcp_url:
config.append(("mcp_servers.rpent.url", mcp_url))
+ # Allow the server to cancel and return its result before the client's
+ # transport deadline. The outer planner deadline still applies.
+ config.append(("mcp_servers.rpent.tool_timeout_sec", mcp_tool_timeout_s() + 5))
if base_url:
normalized = base_url.rstrip("/")
if not normalized.endswith("/v1"):
diff --git a/rpent/planner/utils/http_mcp_server.py b/rpent/planner/utils/http_mcp_server.py
index 73ce53803..c12f17a12 100644
--- a/rpent/planner/utils/http_mcp_server.py
+++ b/rpent/planner/utils/http_mcp_server.py
@@ -34,10 +34,15 @@
import asyncio
import base64
+import math
+import os
import socket
import threading
+from contextlib import suppress
+from functools import partial
from typing import Any
+import anyio
import httpx
import uvicorn
from mcp import types
@@ -72,6 +77,14 @@ def mcp_result(result: ToolResult) -> dict[str, Any]:
}
+def mcp_tool_timeout_s() -> float:
+ """Return the server-owned deadline shared with the Codex client config."""
+ seconds = float(os.environ.get("CODEX_MCP_TOOL_TIMEOUT_S", "86400"))
+ if not math.isfinite(seconds) or seconds <= 0:
+ raise ValueError("CODEX_MCP_TOOL_TIMEOUT_S must be finite and positive")
+ return seconds
+
+
def _strip_mcp_prefix(name: str) -> str:
"""``mcp__rpent__mcp_list_dir`` -> ``mcp_list_dir`` ; passthrough."""
prefix = f"mcp__{SERVER_NAME}__"
@@ -84,6 +97,7 @@ def build_mcp_server(toolkit: Toolkit) -> Server:
"""Expose native declarations through MCP, with Toolkit owning validation."""
mcp_app: Server = Server(SERVER_NAME, version="0.1.0")
tool_execution_lock = asyncio.Lock()
+ tool_timeout_s = mcp_tool_timeout_s()
@mcp_app.list_tools()
async def _list_tools() -> list[types.Tool]:
@@ -101,10 +115,57 @@ async def _list_tools() -> list[types.Tool]:
@mcp_app.call_tool(validate_input=False)
async def _call_tool(name: str, arguments: dict[str, Any]) -> types.CallToolResult:
lookup = _strip_mcp_prefix(name)
- async with tool_execution_lock:
- tr = await asyncio.get_running_loop().run_in_executor(
- None, toolkit.execute_tool, lookup, arguments or {}
+ request = mcp_app.request_context.request
+ cancel_event = threading.Event()
+
+ def expire() -> None:
+ logger.warning(
+ "MCP tool %s exceeded %.3fs; cancelling", lookup, tool_timeout_s
)
+ cancel_event.set()
+
+ # Some clients report a timeout without disconnecting or sending a
+ # cancellation notification. The server must retire its own operation.
+ deadline = asyncio.get_running_loop().call_later(tool_timeout_s, expire)
+
+ async def watch_disconnect() -> None:
+ # The transport has consumed the body. JSON responses do not watch
+ # for disconnects while awaiting a tool, unlike SSE responses.
+ while True:
+ message = await request.receive()
+ if message["type"] == "http.disconnect":
+ cancel_event.set()
+ return
+
+ watcher = asyncio.create_task(watch_disconnect())
+ try:
+ async with tool_execution_lock:
+ worker = asyncio.get_running_loop().run_in_executor(
+ None,
+ partial(
+ toolkit.execute_tool,
+ lookup,
+ arguments or {},
+ cancel_event=cancel_event,
+ ),
+ )
+ try:
+ tr = await asyncio.shield(worker)
+ except asyncio.CancelledError:
+ cancel_event.set()
+ # Cancelling an await cannot stop a synchronous robot call.
+ # Keep ownership until it reaches a safe cancellation boundary.
+ with anyio.CancelScope(shield=True):
+ while not worker.done():
+ with suppress(asyncio.CancelledError):
+ await asyncio.shield(worker)
+ worker.result()
+ raise
+ finally:
+ deadline.cancel()
+ watcher.cancel()
+ with anyio.CancelScope(shield=True), suppress(asyncio.CancelledError):
+ await watcher
return types.CallToolResult(**mcp_result(tr))
return mcp_app
@@ -114,7 +175,9 @@ def _build_asgi_app(toolkit: Toolkit) -> Any:
"""Build a raw ASGI3 app wrapping an MCP ``Server`` + streamable HTTP."""
session_manager = StreamableHTTPSessionManager(
app=build_mcp_server(toolkit),
- stateless=True,
+ # Cancellation arrives as a separate POST and must reach the session
+ # that owns the original call, even when its HTTP connection stays open.
+ stateless=False,
json_response=True,
)
@@ -132,7 +195,7 @@ async def asgi_app(scope: dict[str, Any], receive: Any, send: Any) -> None:
# protocol violation into a silent mis-handle.
if shutdown_event["type"] != "lifespan.shutdown":
break
- await send({"type": "lifespan.shutdown.complete"})
+ await send({"type": "lifespan.shutdown.complete"})
break
elif event["type"] == "lifespan.shutdown":
await send({"type": "lifespan.shutdown.complete"})
@@ -174,6 +237,10 @@ def _wait_for_ready(url: str, *, timeout_s: float) -> None:
raise RuntimeError(
f"HttpMcpServer not ready: code: {resp.status_code}; content: {body_preview}"
)
+ c.delete(
+ url,
+ headers={"Mcp-Session-Id": resp.headers["Mcp-Session-Id"]},
+ ).raise_for_status()
class HttpMcpServer:
@@ -240,5 +307,6 @@ def stop(self, *, timeout_s: float = 5.0) -> None:
"leaving references intact",
timeout_s,
)
+ return
self._server = None
self._thread = None
diff --git a/rpent/robots/components/pi05_vla_client.py b/rpent/robots/components/pi05_vla_client.py
index baa14fabe..7b69d903f 100644
--- a/rpent/robots/components/pi05_vla_client.py
+++ b/rpent/robots/components/pi05_vla_client.py
@@ -236,6 +236,10 @@ def encode_obs(self, env_obs: dict) -> dict:
"""Dispatch to the embodiment's encoder from ``_ENCODE_OBS``."""
return _ENCODE_OBS[self._embodiment](env_obs)
+ def status(self, *, timeout_s: float = 10) -> dict[str, Any]:
+ """Query read-only metadata from a server supporting ``vla.status``."""
+ return self._client.call("vla.status", timeout_s=timeout_s)
+
# ---- inference ----
def predict(self, env_obs: dict, options: dict | None = None) -> np.ndarray:
diff --git a/rpent/robots/components/pi05_vla_server.py b/rpent/robots/components/pi05_vla_server.py
index 2517f1699..a5801fbda 100644
--- a/rpent/robots/components/pi05_vla_server.py
+++ b/rpent/robots/components/pi05_vla_server.py
@@ -225,6 +225,17 @@ def __init__(
cfg["model_path"],
)
self._model = get_model(cfg, torch_dtype=None).cuda().eval()
+ parameters = list(self._model.parameters())
+ self._status = {
+ "model_path": model_path,
+ "repo_id": repo_id,
+ "embodiment": embodiment,
+ "dtypes": sorted({str(p.dtype) for p in parameters}),
+ "device": str(parameters[0].device) if parameters else None,
+ "load_seconds": time.time() - t0,
+ "config": OmegaConf.to_container(cfg, resolve=True),
+ }
+ self._rpc["vla.status"] = lambda: self._status
logger.info("model ready in %.1fs", time.time() - t0)
def _register_rpc(self):
diff --git a/rpent/tools/human_in_the_loop.py b/rpent/tools/human_in_the_loop.py
index 6b0126fa0..dde693f2b 100644
--- a/rpent/tools/human_in_the_loop.py
+++ b/rpent/tools/human_in_the_loop.py
@@ -16,18 +16,101 @@
from __future__ import annotations
+import json
import queue
import select
import sys
import threading
import uuid
from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
from rpent.utils.logging import get_logger
logger = get_logger("human_in_the_loop")
+@dataclass(frozen=True)
+class OperatorCommand:
+ """Parsed operator input, without granting any scene or motion permission."""
+
+ name: str
+ notes: str = ""
+ request_id: str | None = None
+ answer: str = ""
+
+
+def parse_operator_command(line: str) -> OperatorCommand | None:
+ """Parse the shared terminal/Web commands; ordinary chat returns None."""
+ parts = line.strip().split(maxsplit=1)
+ command = parts[0].lower() if parts else ""
+ if command == "/succes":
+ command = "/success"
+ if command not in {
+ "/done",
+ "/continue",
+ "/success",
+ "/failure",
+ "/abort",
+ "/operator",
+ }:
+ return None
+ notes = parts[1] if len(parts) > 1 else ""
+ if command == "/operator":
+ tokens = notes.split(maxsplit=2)
+ if len(tokens) < 2:
+ raise ValueError("Usage: /operator [notes]")
+ return OperatorCommand(
+ name="operator",
+ request_id=tokens[0],
+ answer=tokens[1],
+ notes=tokens[2] if len(tokens) > 2 else "",
+ )
+ return OperatorCommand(name=command[1:], notes=notes)
+
+
+def record_operator_outcome(
+ *,
+ memory_root: Path,
+ recipe_tag: str,
+ run_name: str,
+ session_number: int,
+ state_output_dir: str | Path,
+ verdict: dict[str, Any],
+ messages: list[dict[str, Any]],
+ planner_error: str | None,
+) -> Path:
+ """Archive an operator outcome without overwriting a previous attempt."""
+ outcome = verdict["operator_verdict"]
+ path = (
+ memory_root
+ / "_internal"
+ / "inbox"
+ / recipe_tag
+ / "wip"
+ / f"{run_name}-session-{session_number:03d}-{outcome}.json"
+ )
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("x") as file:
+ json.dump(
+ {
+ "verdict": verdict,
+ "session": session_number,
+ "evidence_dir": str(state_output_dir),
+ "messages": messages,
+ "planner_error": planner_error,
+ "note": "Operator-confirmed outcome; explanations still require evidence review.",
+ },
+ file,
+ indent=2,
+ default=str,
+ )
+ logger.info("Attempt evidence saved: %s", path)
+ return path
+
+
class HumanInTheLoopInput:
"""Use the existing interactive reader, or read an otherwise unowned TTY.
@@ -36,40 +119,62 @@ class HumanInTheLoopInput:
"""
help_text = """Human-interactive exploration commands:
- /done Confirm the pending scene reset.
- /continue Continue from a pending operator verdict.
- /success Finish successfully and save exploration memory.
- /failure Finish with a failure record.
- /abort Abort exploration without publishing success memory.
+ /done [notes] Confirm the pending scene reset, optionally describing the scene.
+ /continue [notes] Continue from a pending verdict with optional instructions.
+ /success [notes] Record this successful attempt and continue in a new session.
+ /failure [notes] Record this failed attempt and continue in a new session.
+ /abort [notes] Abort exploration without publishing success memory.
Words without / remain normal messages to the agent.
"""
- def __init__(self, *, interactive: bool) -> None:
+ def __init__(self, *, interactive: bool, feedback_path: Path | None = None) -> None:
self.interactive = interactive
+ self._feedback_path = feedback_path
+ self._feedback: list[str] = []
self._lock = threading.Lock()
self._pending: tuple[str, queue.Queue[str | None]] | None = None
self._pending_kind: str | None = None
self._closed = False
- self._verdict_handler: Callable[[str], bool] | None = None
+ self._continue_revision = 0
+ self._verdict_handler: Callable[[str, str], bool] | None = None
- def bind_verdict(self, handler: Callable[[str], bool] | None) -> None:
+ def bind_verdict(self, handler: Callable[[str, str], bool] | None) -> None:
"""Bind program-level verdict control for the active exploration session."""
with self._lock:
self._verdict_handler = handler
def route_line(self, line: str) -> bool:
"""Consume operator commands; return False for ordinary planner input."""
- command = line.strip().lower()
+ try:
+ parsed = parse_operator_command(line)
+ except ValueError as exc:
+ logger.warning("%s", exc)
+ return True
+ if parsed is None:
+ if line.strip() and not line.startswith("/"):
+ with self._lock:
+ self._record_feedback(line.strip())
+ return False
+ command = f"/{parsed.name}"
+ notes = parsed.notes
if command in {"/done", "/continue"}:
expected = "reset" if command == "/done" else "verdict"
with self._lock:
+ if command == "/continue" and self._pending is None:
+ if notes:
+ self._record_feedback(notes)
+ return False
if self._pending is None or self._pending_kind != expected:
logger.warning(
"%s refused: no pending %s request.", command, expected
)
else:
- self._pending[1].put(command[1:])
+ if command == "/continue":
+ self._continue_revision += 1
+ self._pending[1].put(command[1:] + (" " + notes if notes else ""))
+ if notes:
+ self._record_feedback(notes)
self._pending = None
self._pending_kind = None
logger.info("%s accepted.", command)
@@ -78,27 +183,56 @@ def route_line(self, line: str) -> bool:
verdict = command[1:]
with self._lock:
handler = self._verdict_handler
- if handler is None or not handler(verdict):
+ if handler is None or not handler(verdict, notes):
logger.warning(
"/%s refused: no eligible exploration attempt, or a verdict is already closing the run.",
verdict,
)
+ elif notes:
+ with self._lock:
+ self._record_feedback(notes)
return True
- if line.split(maxsplit=1)[:1] != ["/operator"]:
- return False
- parts = line.split(maxsplit=2)
with self._lock:
pending = self._pending
- if pending is None or len(parts) != 3 or parts[1] != pending[0]:
+ if pending is None or parsed.request_id != pending[0]:
logger.warning(
"No matching operator request; use /operator ."
)
else:
- pending[1].put(parts[2])
+ if self._pending_kind == "verdict" and parsed.answer == "continue":
+ self._continue_revision += 1
+ reply = parsed.answer + (" " + notes if notes else "")
+ pending[1].put(reply)
+ self._record_feedback(reply)
self._pending = None
self._pending_kind = None
return True
+ @property
+ def continue_revision(self) -> int:
+ """Count accepted continue replies, including replies during a model turn."""
+ with self._lock:
+ return self._continue_revision
+
+ def _record_feedback(self, text: str) -> None:
+ self._feedback.append(text)
+ if self._feedback_path is not None:
+ self._feedback_path.parent.mkdir(parents=True, exist_ok=True)
+ with self._feedback_path.open("a") as f:
+ f.write(json.dumps({"text": text}, ensure_ascii=False) + "\n")
+ logger.info("[feedback] recorded for exploration handoff: %s", text)
+
+ def feedback_prompt(self) -> str:
+ with self._lock:
+ if not self._feedback:
+ return ""
+ return (
+ "\nOperator feedback, chronological (later task corrections supersede "
+ "earlier task assumptions, not hardware safety requirements):\n"
+ + "\n".join(self._feedback)
+ + "\nAcknowledge the latest exploration focus and adapt your plan.\n"
+ )
+
def close(self) -> None:
"""Release pending input and detach the active verdict handler."""
with self._lock:
@@ -109,6 +243,24 @@ def close(self) -> None:
self._pending[1].put(None)
self._pending = None
+ @property
+ def pending_kind(self) -> str | None:
+ with self._lock:
+ return self._pending_kind
+
+ @property
+ def closed(self) -> bool:
+ with self._lock:
+ return self._closed
+
+ def cancel_pending(self) -> None:
+ """Retire the old request, never confirm it or close the input reader."""
+ with self._lock:
+ if self._pending is not None:
+ self._pending[1].put(None)
+ self._pending = None
+ self._pending_kind = None
+
def __call__(self, prompt: str, check_cancelled: Callable[[], None]) -> str | None:
return self.request(prompt, check_cancelled)
@@ -145,12 +297,15 @@ def request(
"/done" if kind == "reset" else "/continue" if kind == "verdict" else None
)
hint = (
- f"\nShortcut: {shortcut}; /success, /failure or /abort ends exploration."
+ f"\nShortcut: {shortcut}; /success or /failure starts a new session; "
+ "/abort ends exploration."
if shortcut
else ""
)
- print(f"\n{prompt}\nReply: /operator {request_id} {hint}", flush=True)
try:
+ print(
+ f"\n{prompt}\nReply: /operator {request_id} {hint}", flush=True
+ )
while True:
check_cancelled()
try:
diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py
index 5ad493ef7..c0da2b444 100644
--- a/rpent/tools/toolkit.py
+++ b/rpent/tools/toolkit.py
@@ -138,7 +138,13 @@ def list_tools(self) -> tuple[Tool, ...]:
for tool in self._tools.values()
)
- def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult:
+ def execute_tool(
+ self,
+ name: str,
+ input_dict: dict[str, Any],
+ *,
+ cancel_event: threading.Event | None = None,
+ ) -> ToolResult:
"""Validate one call, execute it, and capture state for advancing tools."""
tool = self._tools.get(name)
if tool is None:
@@ -165,12 +171,17 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult:
return ToolResult(
data={"error": "another tool operation is still active"}
)
- operation = _ToolOperation()
+ operation = _ToolOperation(
+ cancel_event=cancel_event
+ if cancel_event is not None
+ else threading.Event()
+ )
self._active_operation = operation
try:
started = time.perf_counter()
try:
+ self.raise_if_cancelled()
native = tool(**kwargs)
except TypeError as exc:
native = ToolResult(
@@ -277,6 +288,13 @@ def raise_if_cancelled(self) -> None:
if operation is not None and operation.cancel_event.is_set():
raise ToolCancelled("tool operation interrupted")
+ def wait_active(self) -> None:
+ """Wait for an already-authorized operation without cancelling it."""
+ with self._operation_lock:
+ operation = self._active_operation
+ if operation is not None:
+ operation.done_event.wait()
+
def close(self) -> None:
"""Release the robot-side primitives / servers at end of run. Default: no-op."""
diff --git a/rpent/utils/config.py b/rpent/utils/config.py
index e8e73bbf7..88c2366e7 100644
--- a/rpent/utils/config.py
+++ b/rpent/utils/config.py
@@ -55,9 +55,9 @@ def get_libero_type() -> str:
return os.environ.get("LIBERO_TYPE", "pro")
-def get_rlinf_repo_path() -> Path:
- """Return the RLinf path that RPC servers import."""
+def get_rlinf_repo_path() -> Path | None:
+ """Return an explicit RLinf source override, otherwise use the installed package."""
env = os.environ.get("RPENT_RLINF_ROOT") or os.environ.get("RLINF_REPO_PATH")
if env:
return Path(env).expanduser().resolve()
- return (get_repo_root().parent / "rlinf").resolve()
+ return None
diff --git a/rpent/utils/daemon.py b/rpent/utils/daemon.py
index eb7b3c069..ee1de2dcd 100644
--- a/rpent/utils/daemon.py
+++ b/rpent/utils/daemon.py
@@ -115,12 +115,15 @@ def __init__(
self.cmd = cmd
self.subprocess_env = os.environ.copy()
self.subprocess_env.update(env_overrides or {})
- # Put the RLinf checkout first on the child's PYTHONPATH
- rlinf_path = str(get_rlinf_repo_path())
- pythonpath = self.subprocess_env.get("PYTHONPATH", "")
- self.subprocess_env["PYTHONPATH"] = (
- f"{rlinf_path}{os.pathsep}{pythonpath}" if pythonpath else rlinf_path
- )
+ # Source checkouts are opt-in; normal deployments use installed RLinf.
+ rlinf_path = get_rlinf_repo_path()
+ if rlinf_path is not None:
+ pythonpath = self.subprocess_env.get("PYTHONPATH", "")
+ self.subprocess_env["PYTHONPATH"] = (
+ f"{rlinf_path}{os.pathsep}{pythonpath}"
+ if pythonpath
+ else str(rlinf_path)
+ )
self.log_path = log_path
self.cwd = cwd
self._proc: subprocess.Popen | None = None
diff --git a/tests/e2e_tests/dual_franka/dual_franka_vla.py b/tests/e2e_tests/dual_franka/dual_franka_vla.py
index 9b0ddfa5a..076edb4b7 100644
--- a/tests/e2e_tests/dual_franka/dual_franka_vla.py
+++ b/tests/e2e_tests/dual_franka/dual_franka_vla.py
@@ -104,10 +104,13 @@ def __init__(
workspace: dict[str, Any],
prompt: str,
expected_steps: int,
+ *,
+ status: dict[str, Any] | None = None,
) -> None:
self.env, self.model = env, model
self.output, self.workspace = Path(output), workspace
self.prompt = prompt
+ self.status = status
self.index = 0
self.execution_uncertain = False
self.episode_done = False
@@ -181,9 +184,7 @@ def run_console(
emit: Callable[[str], None] = print,
) -> None:
"""Read explicit commands without invoking an Agent."""
- help_text = (
- "prompt | infer | step | run N (1..20 chunks) | reset | quit"
- )
+ help_text = "status | prompt | infer | step | run N (1..20 chunks) | reset | quit"
emit(help_text)
emit("infer 不下发预测动作；step 执行一个动作块。Ctrl-C 退出会话，不等同硬件急停。")
while True:
@@ -199,6 +200,18 @@ def run_console(
raise ValueError("prompt cannot be empty")
session.prompt = arg.strip()
emit("Prompt: " + session.prompt)
+ elif command == "status" and not arg:
+ session.status = session.model.status(timeout_s=10)
+ emit(
+ json.dumps(
+ {
+ **session.status,
+ "prompt": session.prompt,
+ "expected_action_steps": session.expected_steps,
+ },
+ ensure_ascii=False,
+ )
+ )
elif command == "help" and not arg:
emit(help_text)
elif command == "reset" and not arg:
@@ -233,7 +246,7 @@ def run_console(
def run_session(args: argparse.Namespace) -> int:
"""Own the diagnostic runtime and close it when the console exits."""
- if args.expected_action_steps <= 0:
+ if args.expected_action_steps is not None and args.expected_action_steps <= 0:
raise ValueError("expected action steps must be positive")
spec = get_robot_spec()
from robots.dual_franka.runtime_config import DEFAULT_CONFIG
@@ -261,10 +274,18 @@ def run_session(args: argparse.Namespace) -> int:
args, output, NullDashboardEventSink(), {"env", "vla"}
)
model = kwargs["model"]
+ expected_steps = args.expected_action_steps
+ status = None
+ if expected_steps is None:
+ status = model.status(timeout_s=10)
+ expected_steps = int(status["config"]["openpi"]["action_chunk"])
+ if expected_steps <= 0:
+ raise ValueError("expected action steps must be positive")
save_record(
output / "deployment",
{
- "expected_action_steps": args.expected_action_steps,
+ "expected_action_steps": expected_steps,
+ "server": status,
"robot_config": data,
"camera_meta": kwargs["env"].get_camera_meta(),
},
@@ -275,7 +296,8 @@ def run_session(args: argparse.Namespace) -> int:
output,
data["workspace"],
args.instruction,
- args.expected_action_steps,
+ expected_steps,
+ status=status,
)
logger.info("Records: %s", output)
run_console(session)
@@ -298,8 +320,11 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument(
"--expected-action-steps",
type=int,
- default=20,
- help="Expected prediction chunk length for action validation (default: 20)",
+ default=None,
+ help=(
+ "Override prediction chunk length; otherwise read vla.status "
+ "from the server. Use an explicit value with older servers."
+ ),
)
parser.add_argument("--output-dir", default=None)
return parser
diff --git a/tests/integration_tests/robots/dual_franka/test_rlinf_runtime.py b/tests/integration_tests/robots/dual_franka/test_rlinf_runtime.py
new file mode 100644
index 000000000..65f56b31d
--- /dev/null
+++ b/tests/integration_tests/robots/dual_franka/test_rlinf_runtime.py
@@ -0,0 +1,376 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Offline contracts using installed official RLinf, without Ray or hardware.
+
+Install the pinned official RLinf dependency, then run from the RPent root::
+
+ RPENT_RUN_RLINF_CONTRACT=1 python -m pytest \
+ tests/integration_tests/robots/dual_franka/test_rlinf_runtime.py -v
+
+The real environment, observation conversion and Gym registration are exercised.
+Only camera/arm/hand readings are faked; hardware setup, connections and Ray
+startup fail immediately if accidentally reached. This is not a live-robot test.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import os
+import queue
+from concurrent.futures import ThreadPoolExecutor
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+
+pytestmark = pytest.mark.skipif(
+ os.environ.get("RPENT_RUN_RLINF_CONTRACT") != "1",
+ reason="set RPENT_RUN_RLINF_CONTRACT=1 with the official RLinf installed",
+)
+
+
+@pytest.fixture(autouse=True)
+def no_hardware_or_ray(monkeypatch):
+ import ray
+ from rlinf.envs.real.env import RealWorldEnv
+ from rlinf.envs.real.franka.dual_base import DualFrankaEnv
+ from rlinf.robotics.parts.base import Connection
+
+ def forbidden(*_args, **_kwargs):
+ pytest.fail("offline RLinf contract attempted hardware or Ray startup")
+
+ monkeypatch.setattr(ray, "init", forbidden)
+ monkeypatch.setattr(RealWorldEnv, "__init__", forbidden)
+ monkeypatch.setattr(RealWorldEnv, "realworld_setup", forbidden)
+ monkeypatch.setattr(Connection, "connect", forbidden)
+ monkeypatch.setattr(DualFrankaEnv, "_setup_hardware", forbidden)
+ monkeypatch.setattr(DualFrankaEnv, "_go_to_rest", forbidden)
+
+
+@pytest.fixture
+def runtime():
+ from robots.dual_franka.runtime_config import load_runtime_config
+
+ return load_runtime_config(None, task_description="offline adapter contract")
+
+
+@pytest.fixture
+def dummy_env(runtime):
+ import gymnasium as gym
+ from omegaconf import OmegaConf
+ from rlinf.robotics.discovery import RobotInfo
+ from rlinf.robotics.robots.dual_franka import DualFrankaConfig
+
+ from robots.dual_franka.rpent_env import register_rpent_dual_franka_env
+
+ hardware = DualFrankaConfig(
+ **OmegaConf.to_container(
+ runtime.rlinf.cluster.node_groups[0].hardware.configs[0]
+ )
+ )
+ override = OmegaConf.to_container(runtime.rlinf.env.eval.override_cfg)
+ override["is_dummy"] = True
+ register_rpent_dual_franka_env()
+ env = gym.make(
+ runtime.rlinf.env.eval.init_params.id,
+ override_cfg=override,
+ worker_info=None,
+ robot_info=RobotInfo(type="DualFranka", model="DualFranka", config=hardware),
+ env_idx=0,
+ env_cfg=OmegaConf.to_container(runtime.rlinf.env.eval),
+ )
+ try:
+ yield env
+ finally:
+ env.close()
+
+
+class _CameraReading:
+ def __init__(self, info, value):
+ self.camera_info = info
+ self.depth_scale = 0.001
+ self.calls = 0
+ self.fail = False
+ self.frame = np.full((4, 6, 3), [value, value + 1, value + 2], dtype=np.uint8)
+ self.depth = np.full((4, 6), 0.42, dtype=np.float32)
+
+ def get_observation(self, **_kwargs):
+ self.calls += 1
+ if self.fail:
+ raise queue.Empty
+ reading = {"frame": self.frame.copy()}
+ if self.camera_info.enable_depth:
+ reading["depth"] = self.depth.copy()
+ return reading
+
+
+@pytest.fixture
+def reading_env(dummy_env):
+ from rlinf.robotics.parts.arms.franka import FrankaRobotState
+
+ env = dummy_env.unwrapped
+ env.config.is_dummy = False
+ env._cameras = {
+ info.name: _CameraReading(
+ dataclasses.replace(info, enable_depth=True), value=10 + index * 10
+ )
+ for index, info in enumerate(env._camera_infos())
+ }
+ env._last_camera_frame = {}
+ env.camera_player = SimpleNamespace(
+ put_frame=lambda _frames: None, stop=lambda: None
+ )
+ left = FrankaRobotState(tcp_pose=np.array([0.4, -0.2, 0.3, 0, 0, 0, 1]))
+ right = FrankaRobotState(tcp_pose=np.array([0.5, 0.2, 0.4, 0, 0, 0, 1]))
+ env._left_arm = SimpleNamespace(get_state=lambda: left)
+ env._right_arm = SimpleNamespace(get_state=lambda: right)
+ env._left_hand = SimpleNamespace(position=0.25, is_open=False)
+ env._right_hand = SimpleNamespace(position=0.75, is_open=True)
+ env._arm_executors = (
+ ThreadPoolExecutor(max_workers=1),
+ ThreadPoolExecutor(max_workers=1),
+ )
+ env.get_tcp_pose()
+ return env
+
+
+def test_official_config_and_registered_dummy_environment(runtime, dummy_env):
+ from rlinf.envs.real.franka.dual_franka_tcp import DualFrankaTCPEnv
+ from rlinf.robotics.robots.dual_franka import DualFrankaConfig
+
+ from robots.dual_franka.rpent_env import RPentDualFrankaEnv
+
+ assert not hasattr(DualFrankaTCPEnv, "get_raw_camera_snapshot")
+ assert "left_camera_node_rank" not in {
+ field.name for field in dataclasses.fields(DualFrankaConfig)
+ }
+ assert isinstance(dummy_env.unwrapped, RPentDualFrankaEnv)
+ assert isinstance(dummy_env.unwrapped, DualFrankaTCPEnv)
+ assert dummy_env.action_space.shape == (20,)
+ obs, _ = dummy_env.reset(seed=7)
+ assert dummy_env.observation_space.contains(obs)
+ assert obs["state"]["tcp_pose_rot6d"].shape == (18,)
+ assert set(obs["frames"]) == {"base_0_rgb", "left_wrist_0_rgb", "right_wrist_0_rgb"}
+ assert runtime.rlinf.env.eval.auto_reset is False
+ _, _, _, _, _ = dummy_env.step(dummy_env.action_space.sample())
+
+
+def test_raw_snapshot_matches_official_policy_processing(reading_env):
+ from rlinf.envs.real.franka.dual_franka_tcp import DualFrankaTCPEnv
+
+ env = reading_env
+ frames, depths = env._get_camera_observation()
+ snapshot = env.get_raw_camera_snapshot()
+ assert all(camera.calls == 1 for camera in env._cameras.values())
+ for name, camera in env._cameras.items():
+ np.testing.assert_array_equal(
+ snapshot["raw_frames"][name], camera.frame[..., ::-1]
+ )
+ np.testing.assert_array_equal(snapshot["raw_depths"][name], camera.depth)
+ assert snapshot["camera_meta"][name]["depth_unit"] == "m"
+
+ original_frames, original_depths = DualFrankaTCPEnv._get_camera_observation(env)
+ for name in frames:
+ np.testing.assert_array_equal(frames[name], original_frames[name])
+ np.testing.assert_array_equal(depths[name], original_depths[name])
+ name = next(iter(frames))
+ snapshot["raw_frames"][name][:] = 0
+ assert np.any(env.get_raw_camera_snapshot()["raw_frames"][name])
+ assert all(camera.calls == 2 for camera in env._cameras.values())
+
+
+def test_camera_recovery_never_reuses_stale_projection_snapshot(reading_env):
+ env = reading_env
+ with pytest.raises(RuntimeError, match="No fresh complete camera snapshot"):
+ env.get_raw_camera_snapshot()
+ env._get_camera_observation()
+ camera = next(iter(env._cameras.values()))
+ camera.fail = True
+ policy_frames, _ = env._get_camera_observation()
+ assert policy_frames
+ with pytest.raises(RuntimeError, match="No fresh complete camera snapshot"):
+ env.get_raw_camera_snapshot()
+ camera.fail = False
+ camera.frame[:] = 77
+ env._get_camera_observation()
+ name = camera.camera_info.name
+ assert np.all(env.get_raw_camera_snapshot()["raw_frames"][name] == 77)
+
+
+@pytest.mark.parametrize("with_depth", [False, True])
+def test_extra_camera_uses_official_depth_conversion_once(monkeypatch, with_depth):
+ from rlinf.robotics.parts.cameras import CameraInfo, RealSenseCamera
+
+ from robots.dual_franka.env_server import _create_worker_class
+ from robots.franka import rpent_env
+
+ camera = RealSenseCamera(
+ CameraInfo("extra_rgb", "offline-camera", enable_depth=with_depth)
+ )
+ camera._depth_scale = 0.001
+ frame = np.full((4, 6, 3), [10, 20, 30], dtype=np.uint8)
+ if with_depth:
+ frame = np.concatenate(
+ [frame, np.full((4, 6, 1), 420, dtype=np.float32)], axis=-1
+ )
+ calls = []
+
+ def get_frame(**kwargs):
+ calls.append(kwargs["timeout"])
+ return frame.copy()
+
+ monkeypatch.setattr(camera, "get_frame", get_frame)
+ intrinsics = {"width": 6, "height": 4, "fx": 10.0, "fy": 11.0}
+ monkeypatch.setattr(rpent_env, "realsense_color_intrinsics", lambda _: intrinsics)
+ worker = object.__new__(_create_worker_class())
+ worker._perception_cameras = {"extra": camera}
+ worker._perception_camera_meta = {}
+
+ result = worker._capture_perception_camera_snapshot()
+
+ assert calls == [2]
+ rgb = result["raw_frames"]["extra_rgb"]
+ assert rgb.shape == (4, 6, 3)
+ assert rgb.dtype == np.uint8
+ assert rgb[0, 0].tolist() == [30, 20, 10]
+ assert result["camera_meta"]["extra_rgb"]["color_intrinsics"] == intrinsics
+ if with_depth:
+ depth = result["raw_depths"]["extra_rgb"]
+ assert depth.shape == (4, 6)
+ assert depth.dtype == np.float32
+ np.testing.assert_allclose(depth, 0.42)
+ else:
+ assert result["raw_depths"] == {}
+ assert result["camera_meta"]["extra_rgb"]["depth_available"] is False
+
+
+def test_worker_observation_uses_official_wrapper_and_latest_acquisition(reading_env):
+ import gymnasium as gym
+ from rlinf.envs.real.env import RealWorldEnv
+
+ from robots.dual_franka.env_server import _create_worker_class
+
+ env = reading_env
+ vector_env = gym.vector.SyncVectorEnv([lambda: env])
+ # Bypass only the process/hardware-owning constructor, not conversion logic.
+ wrapped = object.__new__(RealWorldEnv)
+ wrapped.env = vector_env
+ wrapped.main_image_key = "left_wrist_0_rgb"
+ wrapped.task_descriptions = [env.task_description]
+ worker = object.__new__(_create_worker_class())
+ worker.env = wrapped
+ worker._perception_cameras = {}
+ worker._perception_camera_meta = {}
+ try:
+ obs = worker.get_observation()
+ assert obs["states"].shape == (20,)
+ np.testing.assert_allclose(obs["states"][:2], [0.25, 0.75])
+ assert obs["main_images"].shape == (224, 224, 3)
+ assert obs["extra_view_images"].shape == (2, 224, 224, 3)
+ assert obs["main_depths"].shape == (224, 224)
+ assert all(camera.calls == 1 for camera in env._cameras.values())
+ for camera in env._cameras.values():
+ camera.frame[:] = 99
+ refreshed = worker.get_observation()
+ assert np.all(refreshed["main_images"] == 99)
+ assert all(
+ np.all(frame == 99) for frame in refreshed["raw_camera_frames"].values()
+ )
+ assert all(camera.calls == 2 for camera in env._cameras.values())
+ assert not np.all(obs["main_images"] == 99)
+ finally:
+ vector_env.close()
+
+
+@pytest.mark.parametrize("is_open", [False, True])
+def test_official_zero_gripper_command_leaves_either_hand_state_unchanged(
+ reading_env, is_open
+):
+ def forbidden():
+ pytest.fail("zero gripper command issued an open/close action")
+
+ hand = SimpleNamespace(is_open=is_open, open=forbidden, close=forbidden)
+ assert reading_env._gripper_action(0, hand, 0.0) is False
+ assert hand.is_open is is_open
+
+
+def test_remote_camera_metadata_uses_official_proxy_without_local_sdk_profile(
+ dummy_env,
+):
+ from rlinf.robotics.parts.cameras import Camera, CameraInfo, RealSenseCamera
+ from rlinf.robotics.placement.handles import PartWorkerHost, remote_view_of
+
+ from robots.dual_franka.rpent_env import RPentRealSenseCamera
+
+ info = CameraInfo(
+ name="right_wrist_0_rgb",
+ serial_number="remote-camera",
+ camera_type="rpent_realsense",
+ enable_depth=True,
+ )
+ intrinsics = {
+ "width": 640,
+ "height": 480,
+ "fx": 600.0,
+ "fy": 601.0,
+ "ppx": 320.0,
+ "ppy": 240.0,
+ }
+ calls = []
+
+ class Reply:
+ def __init__(self, value):
+ self.value = value
+
+ def wait(self):
+ return [self.value]
+
+ class MetadataTransport:
+ def attribute(self, name):
+ calls.append(("attribute", name))
+ return Reply({"camera_info": info, "depth_scale": 0.001}[name])
+
+ def get_color_intrinsics(self):
+ calls.append(("method", "get_color_intrinsics"))
+ return Reply(intrinsics)
+
+ camera = RPentRealSenseCamera(info, node_rank=1)
+ camera._group = MetadataTransport()
+ # Connection.connect changes the declared part to this same official view;
+ # only the worker-group transport is replaced, without opening hardware.
+ camera.__class__ = remote_view_of(RPentRealSenseCamera)
+ assert isinstance(camera, RPentRealSenseCamera)
+ assert not hasattr(camera, "profile")
+ assert Camera.backend("realsense") is RealSenseCamera
+ worker_cls = PartWorkerHost.worker_class(RPentRealSenseCamera)
+ assert "get_color_intrinsics" in worker_cls.__dict__
+
+ env = dummy_env.unwrapped
+ env._cameras = {info.name: camera}
+ env._rpent_camera_snapshot = {
+ "raw_frames": {info.name: np.zeros((480, 640, 3), dtype=np.uint8)},
+ "raw_depths": {info.name: np.full((480, 640), 0.4, dtype=np.float32)},
+ }
+ metadata = env.get_raw_camera_metadata()[info.name]
+ assert metadata["serial_number"] == "remote-camera"
+ assert metadata["camera_type"] == "realsense"
+ assert metadata["depth_scale"] == 0.001
+ assert metadata["color_intrinsics"] == intrinsics
+ assert metadata["projection_supported"] is True
+ assert calls == [
+ ("attribute", "camera_info"),
+ ("method", "get_color_intrinsics"),
+ ("attribute", "depth_scale"),
+ ]
diff --git a/tests/manual/__init__.py b/tests/manual/__init__.py
new file mode 100644
index 000000000..66c1fc810
--- /dev/null
+++ b/tests/manual/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Explicitly invoked manual diagnostics; never automatic hardware tests."""
diff --git a/tests/manual/dual_franka_full_vla.py b/tests/manual/dual_franka_full_vla.py
new file mode 100644
index 000000000..5975bdf30
--- /dev/null
+++ b/tests/manual/dual_franka_full_vla.py
@@ -0,0 +1,217 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Manual-only continuous VLA rollout; never registered with the agent toolkit."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import signal
+import threading
+import time
+from contextlib import ExitStack
+from datetime import datetime
+from pathlib import Path
+
+import yaml
+
+from robots.dual_franka.env_client import DualFrankaEnvClient
+from robots.dual_franka.runtime_config import DEFAULT_CONFIG
+from robots.dual_franka.tasks import get_dual_franka_task
+from rpent.robots.components.pi05_vla_client import Pi05VLAClient
+from rpent.utils.rpc import make_rpc_client
+from tests.e2e_tests.dual_franka.dual_franka_vla import (
+ validate_actions,
+ validate_observation,
+)
+
+
+def run_full_vla(env, model, *, prompt, workspace, expected_steps, stopped, emit):
+ """Run indefinitely, checking cancellation before each single-action RPC.
+
+ An in-flight inference/RPC is allowed to return; its remaining actions are
+ discarded on cancellation. No reset, recovery or gripper command is added.
+ """
+ if not prompt.strip():
+ raise ValueError("policy instruction must not be empty")
+ chunks = steps = 0
+ started = time.monotonic()
+ summary = {"status": "stopped", "reason": "operator_stop"}
+ try:
+ while not stopped():
+ observation = dict(env.get_observation())
+ validate_observation(observation)
+ if stopped():
+ break
+ observation["task_descriptions"] = prompt
+ prediction = model.predict(observation, options={"mode": "eval"})
+ if stopped():
+ break
+ actions = validate_actions(prediction, workspace, expected_steps)
+ chunks += 1
+ emit({"kind": "prediction", "chunk": chunks, "actions": actions.tolist()})
+ for index, action in enumerate(actions):
+ if stopped():
+ return summary
+ # Do not queue a whole prediction chunk on the robot server.
+ result = env.chunk_step(action[None, :])
+ steps += 1
+ emit(
+ {
+ "kind": "action",
+ "chunk": chunks,
+ "index": index,
+ "steps": steps,
+ "terminated": bool(result.get("terminated")),
+ "truncated": bool(result.get("truncated")),
+ }
+ )
+ if result.get("error") or result.get("ok") is False:
+ raise RuntimeError(f"environment action failed: {result}")
+ if result.get("terminated") or result.get("truncated"):
+ summary.update(
+ status="environment_stopped", reason="terminated_or_truncated"
+ )
+ return summary
+ emit(
+ {
+ "kind": "chunk_complete",
+ "chunks": chunks,
+ "steps": steps,
+ "elapsed_s": time.monotonic() - started,
+ }
+ )
+ return summary
+ except BaseException as exc:
+ summary.update(status="failed", reason=f"{type(exc).__name__}: {exc}")
+ raise
+ finally:
+ summary.update(chunks=chunks, steps=steps, elapsed_s=time.monotonic() - started)
+ emit({"kind": "finish", **summary})
+
+
+# Manual-only entry point; not registered among the agent's native tools.
+MANUAL_SKILLS = {"vla_full_rollout": run_full_vla}
+
+
+def build_parser():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--skill", choices=MANUAL_SKILLS, default="vla_full_rollout")
+ parser.add_argument("--env-endpoint", default="http://127.0.0.1:6002")
+ parser.add_argument("--vla-endpoint", default="http://127.0.0.1:6000")
+ parser.add_argument("--task-id", type=int, default=6)
+ parser.add_argument(
+ "--instruction", help="Override task-configured policy instruction"
+ )
+ parser.add_argument("--robot-config", type=Path, default=DEFAULT_CONFIG)
+ parser.add_argument("--output-dir", type=Path)
+ parser.add_argument("--stop-file", type=Path)
+ return parser
+
+
+def main():
+ args = build_parser().parse_args()
+ output = args.output_dir or Path("logs") / datetime.now().strftime(
+ "%Y%m%d-%H%M%S-manual-full-vla"
+ )
+ output = output.resolve()
+ # Never overwrite an earlier experiment's logs or remove its stop signal.
+ output.mkdir(parents=True, exist_ok=False)
+ stop_file = (args.stop_file or output / "STOP").resolve()
+ if stop_file.exists():
+ raise RuntimeError(f"stop file already exists: {stop_file}; use a new path")
+ prompt = args.instruction or get_dual_franka_task(args.task_id).vla_instruction
+ if not prompt:
+ raise ValueError("selected task has no VLA instruction")
+ config = yaml.safe_load(args.robot_config.read_text())
+ event = threading.Event()
+ signals = []
+
+ def request_stop(signum, _frame):
+ signals.append(signal.Signals(signum).name)
+ event.set()
+
+ previous = {
+ s: signal.signal(s, request_stop) for s in (signal.SIGINT, signal.SIGTERM)
+ }
+ print(f"Logs: {output}\nStop file: {stop_file}", flush=True)
+ print(
+ "No reset. No duration/chunk/semantic-boundary limit. Ctrl+C or create STOP to stop.",
+ flush=True,
+ )
+ print(
+ "An in-flight RPC may finish; this is NOT a hardware emergency stop.",
+ flush=True,
+ )
+ try:
+ with ExitStack() as stack:
+ env_rpc = make_rpc_client(args.env_endpoint)
+ stack.callback(env_rpc.close)
+ model_rpc = make_rpc_client(args.vla_endpoint)
+ stack.callback(model_rpc.close)
+ env = DualFrankaEnvClient(env_rpc, reset_on_connect=False)
+ model = Pi05VLAClient(model_rpc, embodiment="dual_franka")
+ status = model.status(timeout_s=10)
+ (output / "deployment.json").write_text(
+ json.dumps(
+ {
+ "skill": args.skill,
+ "task_id": args.task_id,
+ "prompt": prompt,
+ "model": status,
+ "robot_config": str(args.robot_config.resolve()),
+ "stop_file": str(stop_file),
+ "reset_on_connect": False,
+ },
+ indent=2,
+ ensure_ascii=False,
+ )
+ )
+ with (output / "events.jsonl").open("x", buffering=1) as log:
+
+ def emit(record):
+ log.write(
+ json.dumps(
+ {"timestamp": time.time(), **record}, ensure_ascii=False
+ )
+ + "\n"
+ )
+ if record["kind"] == "finish":
+ (output / "outcome.json").write_text(
+ json.dumps({**record, "signals": signals}, indent=2)
+ )
+ if record["kind"] in {"chunk_complete", "finish"}:
+ print(json.dumps(record, ensure_ascii=False), flush=True)
+
+ result = MANUAL_SKILLS[args.skill](
+ env,
+ model,
+ prompt=prompt,
+ workspace=config["workspace"],
+ expected_steps=int(status["config"]["openpi"]["action_chunk"]),
+ stopped=lambda: event.is_set() or stop_file.exists(),
+ emit=emit,
+ )
+ (output / "outcome.json").write_text(
+ json.dumps({**result, "signals": signals}, indent=2)
+ )
+ return 0 if result["status"] == "stopped" else 1
+ finally:
+ for signum, handler in previous.items():
+ signal.signal(signum, handler)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/unit_tests/calibration_tools/test_collector_session.py b/tests/unit_tests/calibration_tools/test_collector_session.py
new file mode 100644
index 000000000..4ff62e1b8
--- /dev/null
+++ b/tests/unit_tests/calibration_tools/test_collector_session.py
@@ -0,0 +1,505 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Offline regression coverage for interrupted and reviewed calibration sessions."""
+
+from __future__ import annotations
+
+import base64
+import copy
+import hashlib
+import importlib
+import io
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def collector_module(monkeypatch):
+ cv2 = pytest.importorskip("cv2")
+ if not hasattr(cv2, "aruco") or not hasattr(cv2.aruco, "CharucoDetector"):
+ pytest.skip("Calibration requires OpenCV with the ChArUco detector")
+ pytest.importorskip("scipy")
+ root = Path(__file__).resolve().parents[3]
+ monkeypatch.syspath_prepend(str(root / "calibration_tools"))
+ return importlib.import_module("base_handeye_collect")
+
+
+@pytest.fixture
+def config(collector_module):
+ return collector_module.CaptureConfig(
+ arm="left",
+ camera_serial="offline-camera",
+ camera_url="http://offline.invalid/frame",
+ reader_command=("offline-reader",),
+ )
+
+
+@pytest.fixture
+def record(collector_module, config):
+ identity = collector_module.np.eye(4)
+ pose = identity.copy()
+ pose[0, 3] = 0.1
+ target = identity.copy()
+ target[2, 3] = 0.8
+ state = {
+ "O_T_EE": pose.flatten(order="F").tolist(),
+ "F_T_EE": identity.flatten(order="F").tolist(),
+ "dq": [0.0] * 7,
+ "robot_mode": 1,
+ "has_errors": False,
+ }
+ return {
+ "index": 1,
+ "arm": config.arm,
+ "calibration_mode": "eye_to_hand",
+ "board": config.board_metadata,
+ "robot_before": state,
+ "robot_after": copy.deepcopy(state),
+ "T_left_base_ee": pose.tolist(),
+ "T_camera_board": target.tolist(),
+ "camera": {
+ "serial": config.camera_serial,
+ "width": 680,
+ "height": 880,
+ "K": [[800.0, 0.0, 340.0], [0.0, 800.0, 440.0], [0.0, 0.0, 1.0]],
+ "distortion": [0.0] * 5,
+ "distortion_model": "distortion.brown_conrady",
+ "host_received_s": 1.0,
+ },
+ "corners": 20,
+ "markers": 12,
+ "charuco_ids": list(range(20)),
+ "charuco_corners_px": [[float(i % 5), float(i // 5)] for i in range(20)],
+ "reprojection_rms_px": 0.2,
+ "drift_m": 0.0,
+ "drift_rad": 0.0,
+ }
+
+
+def save_record(root, record, index):
+ """Create a complete saved sample, preserving its serialized bytes for checks."""
+ folder = root / f"sample_{index:03d}"
+ folder.mkdir(parents=True)
+ value = copy.deepcopy(record)
+ value["index"] = index
+ (folder / "sample.json").write_text(json.dumps(value), encoding="utf-8")
+ (folder / "color.png").write_bytes(b"original-color-png")
+ (folder / "annotated.png").write_bytes(b"original-annotated-png")
+ return folder
+
+
+def snapshot(folder):
+ return {
+ str(path.relative_to(folder)): path.read_bytes()
+ for path in folder.rglob("*")
+ if path.is_file()
+ }
+
+
+def test_existing_session_requires_explicit_resume(
+ collector_module, config, record, tmp_path
+):
+ root = tmp_path / "session"
+ save_record(root, record, 1)
+ original = snapshot(root)
+ with pytest.raises((ValueError, FileExistsError)):
+ collector_module.Collector(config, root)
+ assert snapshot(root) == original
+
+
+def test_resume_preserves_originals_and_continues_after_gaps(
+ collector_module, config, record, tmp_path
+):
+ root = tmp_path / "session"
+ save_record(root, record, 7)
+ save_record(root, record, 2)
+ original = snapshot(root)
+
+ collector = collector_module.Collector(config, root, resume=True)
+
+ assert [sample["index"] for sample in collector.samples] == [2, 7]
+ assert collector.next_index == 8
+ assert collector.status()["count"] == 2
+ assert collector.status()["samples"] == [
+ {"index": index, "corners": 20, "reprojection_rms_px": 0.2} for index in [2, 7]
+ ]
+ assert snapshot(root) == original
+
+
+@pytest.mark.parametrize(
+ ("field", "value"),
+ [
+ ("arm", "right"),
+ ("calibration_mode", "eye_in_hand"),
+ ("board", {"squares": [7, 8]}),
+ ("index", 99),
+ ],
+)
+def test_resume_rejects_incompatible_sample_metadata(
+ collector_module, config, record, tmp_path, field, value
+):
+ root = tmp_path / "session"
+ folder = save_record(root, record, 1)
+ changed = json.loads((folder / "sample.json").read_text())
+ changed[field] = value
+ (folder / "sample.json").write_text(json.dumps(changed))
+ original = snapshot(root)
+
+ with pytest.raises(ValueError):
+ collector_module.Collector(config, root, resume=True)
+
+ assert snapshot(root) == original
+
+
+@pytest.mark.parametrize("field", ["serial", "K", "F_T_EE"])
+def test_resume_rejects_changed_camera_or_end_effector(
+ collector_module, config, record, tmp_path, field
+):
+ root = tmp_path / "session"
+ save_record(root, record, 1)
+ changed = copy.deepcopy(record)
+ if field == "serial":
+ changed["camera"]["serial"] = "another-camera"
+ elif field == "K":
+ changed["camera"]["K"][0][0] += 10.0
+ else:
+ for key in ("robot_before", "robot_after"):
+ changed[key]["F_T_EE"][12] = 0.05
+ save_record(root, changed, 2)
+
+ with pytest.raises(ValueError):
+ collector_module.Collector(config, root, resume=True)
+
+
+@pytest.mark.parametrize("missing", ["sample.json", "color.png", "annotated.png"])
+def test_resume_rejects_incomplete_sample(
+ collector_module, config, record, tmp_path, missing
+):
+ root = tmp_path / "session"
+ folder = save_record(root, record, 1)
+ (folder / missing).unlink()
+
+ with pytest.raises((ValueError, OSError)):
+ collector_module.Collector(config, root, resume=True)
+
+
+def test_delete_archives_original_files_and_retains_monotonic_ids(
+ collector_module, config, record, tmp_path
+):
+ root = tmp_path / "session"
+ folder = save_record(root, record, 7)
+ original = snapshot(folder)
+ collector = collector_module.Collector(config, root, resume=True)
+
+ result = collector.delete_sample(7)
+
+ archived = root / "excluded" / "sample_007"
+ assert result == {
+ "deleted": True,
+ "index": 7,
+ "count": 0,
+ "archived": str(archived),
+ }
+ assert not folder.exists()
+ assert snapshot(archived) == original
+ assert collector.samples == []
+ assert collector.next_index == 8
+ resumed = collector_module.Collector(config, root, resume=True)
+ assert resumed.samples == []
+ assert resumed.next_index == 8
+
+
+@pytest.mark.parametrize("index", [0, -1, True, "1", None, 99])
+def test_delete_rejects_invalid_or_missing_ids_without_mutation(
+ collector_module, config, record, tmp_path, index
+):
+ root = tmp_path / "session"
+ save_record(root, record, 1)
+ collector = collector_module.Collector(config, root, resume=True)
+ original = snapshot(root)
+
+ with pytest.raises(ValueError):
+ collector.delete_sample(index)
+
+ assert snapshot(root) == original
+ assert [sample["index"] for sample in collector.samples] == [1]
+
+
+def test_archive_conflict_never_overwrites_either_copy(
+ collector_module, config, record, tmp_path
+):
+ root = tmp_path / "session"
+ save_record(root, record, 1)
+ collector = collector_module.Collector(config, root, resume=True)
+ archived = save_record(root / "excluded", record, 1)
+ (archived / "color.png").write_bytes(b"previously-excluded-image")
+ original = snapshot(root)
+
+ with pytest.raises((ValueError, FileExistsError)):
+ collector.delete_sample(1)
+
+ assert snapshot(root) == original
+ assert collector.status()["count"] == 1
+
+
+def test_failed_archive_rename_preserves_active_sample(
+ collector_module, config, record, tmp_path, monkeypatch
+):
+ root = tmp_path / "session"
+ folder = save_record(root, record, 1)
+ collector = collector_module.Collector(config, root, resume=True)
+ original = snapshot(root)
+ rename = Path.rename
+
+ def fail_rename(path, target):
+ if path == folder:
+ raise OSError("simulated archive rename failure")
+ return rename(path, target)
+
+ monkeypatch.setattr(Path, "rename", fail_rename)
+ with pytest.raises(OSError, match="simulated"):
+ collector.delete_sample(1)
+
+ assert snapshot(root) == original
+ assert collector.status()["count"] == 1
+ assert collector.next_index == 2
+
+
+def test_capture_after_resume_and_delete_never_reuses_saved_ids(
+ collector_module, config, record, tmp_path, monkeypatch
+):
+ root = tmp_path / "session"
+ save_record(root, record, 2)
+ save_record(root / "excluded", record, 9)
+ collector = collector_module.Collector(config, root, resume=True)
+ collector.delete_sample(2)
+ original = snapshot(root)
+
+ state = copy.deepcopy(record["robot_before"])
+ state["O_T_EE"][12] = 0.3
+ monkeypatch.setattr(collector, "read_state", lambda: copy.deepcopy(state))
+ monkeypatch.setattr(collector_module.time, "sleep", lambda _: None)
+ board_image = collector.board.generateImage((680, 880), marginSize=40)
+ success, encoded = collector_module.cv2.imencode(".png", board_image)
+ assert success
+
+ def camera_response(*args, **kwargs):
+ camera = copy.deepcopy(record["camera"])
+ camera["host_received_s"] = collector_module.time.time()
+ camera["png_b64"] = base64.b64encode(encoded.tobytes()).decode()
+ return io.BytesIO(json.dumps(camera).encode())
+
+ monkeypatch.setattr(collector_module.urllib.request, "urlopen", camera_response)
+
+ result = collector.sample()
+
+ assert result["accepted"] is True
+ assert result["count"] == 1
+ assert Path(result["saved"]).name == "sample_010"
+ assert collector.samples[0]["index"] == 10
+ assert collector.next_index == 11
+ assert all(
+ (root / name).read_bytes() == content for name, content in original.items()
+ )
+ resumed = collector_module.Collector(config, root, resume=True)
+ assert [sample["index"] for sample in resumed.samples] == [10]
+ assert resumed.next_index == 11
+
+
+def request_handler(collector_module, collector, path, method):
+ """Exercise the real routing methods without binding a network socket."""
+ handler_type = collector_module.make_handler(collector)
+ handler = handler_type.__new__(handler_type)
+ handler.path = path
+ responses = []
+ handler.reply = lambda code, body, kind="application/json": responses.append(
+ (code, body, kind)
+ )
+ handler.send_error = lambda code, *args: responses.append((code, b"", ""))
+ getattr(handler, f"do_{method}")()
+ assert len(responses) == 1
+ return responses[0]
+
+
+def test_http_review_delete_and_capture_share_mutation_lock(
+ collector_module, config, record, tmp_path
+):
+ root = tmp_path / "session"
+ save_record(root, record, 3)
+ collector = collector_module.Collector(config, root, resume=True)
+ code, body, kind = request_handler(
+ collector_module, collector, "/samples/3/annotated.png", "GET"
+ )
+ assert (code, body, kind) == (200, b"original-annotated-png", "image/png")
+
+ with collector.lock:
+ for path in ("/sample", "/samples/3/delete"):
+ code, _, _ = request_handler(collector_module, collector, path, "POST")
+ assert code == 409
+ assert collector.status()["count"] == 1
+
+ code, body, _ = request_handler(
+ collector_module, collector, "/samples/3/delete", "POST"
+ )
+ assert code == 200
+ assert json.loads(body)["deleted"] is True
+ assert collector.status()["count"] == 0
+ assert not collector.lock.locked()
+ code, _, _ = request_handler(
+ collector_module, collector, "/samples/3/annotated.png", "GET"
+ )
+ assert code == 404
+
+
+@pytest.mark.parametrize("index", ["0", "-1", "wrong", "99"])
+def test_http_delete_invalid_id_retains_session(
+ collector_module, config, record, tmp_path, index
+):
+ root = tmp_path / "session"
+ save_record(root, record, 1)
+ collector = collector_module.Collector(config, root, resume=True)
+
+ code, _, _ = request_handler(
+ collector_module, collector, f"/samples/{index}/delete", "POST"
+ )
+
+ assert code in (404, 422)
+ assert collector.status()["count"] == 1
+ assert not collector.lock.locked()
+
+
+@pytest.mark.parametrize("restart", [False, True])
+@pytest.mark.parametrize("changed_field", ["K", "F_T_EE"])
+def test_deleting_all_samples_retains_camera_and_frame_baseline(
+ collector_module, config, record, tmp_path, monkeypatch, restart, changed_field
+):
+ root = tmp_path / "session"
+ save_record(root, record, 1)
+ collector = collector_module.Collector(config, root, resume=True)
+ collector.delete_sample(1)
+ if restart:
+ collector = collector_module.Collector(config, root, resume=True)
+ original = snapshot(root)
+ state = copy.deepcopy(record["robot_before"])
+ camera = copy.deepcopy(record["camera"])
+ if changed_field == "K":
+ camera["K"][0][0] += 20.0
+ expected_error = "Camera K"
+ else:
+ state["F_T_EE"][12] = 0.05
+ expected_error = "末端坐标定义与先前样本不同"
+ monkeypatch.setattr(collector, "read_state", lambda: copy.deepcopy(state))
+ monkeypatch.setattr(collector_module.time, "sleep", lambda _: None)
+
+ def camera_response(*args, **kwargs):
+ camera["host_received_s"] = collector_module.time.time()
+ return io.BytesIO(json.dumps(camera).encode())
+
+ monkeypatch.setattr(collector_module.urllib.request, "urlopen", camera_response)
+
+ with pytest.raises(ValueError, match=expected_error):
+ collector.sample()
+
+ assert collector.samples == []
+ assert collector.next_index == 2
+ assert snapshot(root) == original
+
+
+def test_deleting_sample_invalidates_previously_solved_candidate(
+ collector_module, config, record, tmp_path
+):
+ root = tmp_path / "session"
+ for index in range(1, 11):
+ save_record(root, record, index)
+ pose = collector_module.np.eye(4).tolist()
+ candidate = {
+ "status": "candidate_requires_independent_validation",
+ "arm": "left",
+ "calibration_mode": "eye_to_hand",
+ "sample_count": 10,
+ "selected_method": "PARK",
+ "camera_serial": config.camera_serial,
+ "T_left_base_camera": pose,
+ "T_ee_board": pose,
+ }
+ report = {
+ **candidate,
+ "serial": config.camera_serial,
+ "source_hashes": {
+ str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest()
+ for path in root.glob("sample_*/sample.json")
+ },
+ "methods": {"PARK": {"T_left_base_camera": pose, "T_ee_board": pose}},
+ }
+ candidate_path = root / "candidate.json"
+ candidate_path.write_text(json.dumps(candidate))
+ (root / "quality_report.json").write_text(json.dumps(report))
+ common = importlib.import_module("common")
+ assert common.load_candidate(candidate_path) == candidate
+
+ collector_module.Collector(config, root, resume=True).delete_sample(5)
+
+ with pytest.raises(ValueError, match="Training samples changed"):
+ common.load_candidate(candidate_path)
+ assert json.loads(candidate_path.read_text()) == candidate
+
+
+@pytest.mark.parametrize("resume_without_output", [False, True])
+def test_cli_requires_explicit_existing_session_selection(
+ collector_module,
+ config,
+ record,
+ tmp_path,
+ monkeypatch,
+ capsys,
+ resume_without_output,
+):
+ root = tmp_path / "session"
+ save_record(root, record, 1)
+ original = snapshot(root)
+ arguments = [
+ "base_handeye_collect.py",
+ "--arm",
+ config.arm,
+ "--camera-serial",
+ config.camera_serial,
+ "--camera-url",
+ config.camera_url,
+ "--reader",
+ "offline-reader",
+ "--robot-ip",
+ "192.0.2.1",
+ ]
+ if resume_without_output:
+ arguments.append("--resume")
+ message = "--resume requires --output"
+ else:
+ arguments.extend(["--output", str(root)])
+ message = "already exists"
+ monkeypatch.setattr(sys, "argv", arguments)
+ monkeypatch.setattr(
+ collector_module,
+ "serve",
+ lambda *args: pytest.fail("Must reject before serving"),
+ )
+
+ with pytest.raises(SystemExit) as error:
+ collector_module.main()
+
+ assert error.value.code == 2
+ assert message in capsys.readouterr().err
+ assert snapshot(root) == original
diff --git a/tests/unit_tests/robots/conftest.py b/tests/unit_tests/robots/conftest.py
index 964c3496f..9ea97daed 100644
--- a/tests/unit_tests/robots/conftest.py
+++ b/tests/unit_tests/robots/conftest.py
@@ -150,6 +150,7 @@ def fake_rlinf_realworld_modules(monkeypatch: pytest.MonkeyPatch) -> None:
"ee_pose_limit_max",
]
dual_hardware_fields = [
+ "compliance",
"left_robot_ip",
"right_robot_ip",
"base_camera_serials",
diff --git a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py
new file mode 100644
index 000000000..dcbc9a17b
--- /dev/null
+++ b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py
@@ -0,0 +1,428 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Attended Dashboard reset, verdict and memory lifecycle with fake hardware."""
+
+import json
+
+import pytest
+
+from robots.dual_franka.toolkit import DualFrankaToolkit
+from rpent.dashboard.events import NullDashboardEventSink
+from rpent.memory import MemoryManager
+from tests.unit_tests.robots.dual_franka.test_exploration import FakeEnv
+
+
+class Env(FakeEnv):
+ def get_robot_state(self):
+ return {**super().get_robot_state(), "coordinate_frame": "right_base"}
+
+
+@pytest.fixture
+def build(tmp_path, monkeypatch, dual_franka_robot_config):
+ from robots.franka.runtime_config import set_robot_config_path
+
+ set_robot_config_path(dual_franka_robot_config)
+ monkeypatch.setattr("robots.dual_franka.toolkit.get_output_dir", lambda: tmp_path)
+ created = []
+
+ def make(*, confirm=True):
+ root = tmp_path / str(len(created))
+ env, replies = Env(), ["done"]
+ toolkit = DualFrankaToolkit(
+ runtime_kwargs={"env": env, "model": None, "task_description": "test"},
+ dashboard_events=NullDashboardEventSink(),
+ memory=MemoryManager(
+ root / "memory", memory_access="inbox_write", inbox_cell_tag="test"
+ ),
+ mode="evaluation",
+ state_output_dir=root,
+ operator_input=lambda *a, **kw: replies.pop(0),
+ )
+ created.append(toolkit)
+ if confirm:
+ result = call(toolkit, "request_scene_reset", reason="operator ready")
+ assert result["operator_lifecycle"]["scene_ready"]
+ return toolkit, env, replies
+
+ yield make
+ for toolkit in created:
+ toolkit.close()
+
+
+def call(toolkit, tool, **kwargs):
+ return toolkit.execute_tool(tool, kwargs).data
+
+
+@pytest.mark.parametrize("reply", ["done", "abort"])
+@pytest.mark.parametrize("shortcut", [False, True])
+@pytest.mark.parametrize("notes", ["", "calibration board still attached"])
+def test_dashboard_operator_reset_is_request_scoped(
+ build, tmp_path, reply, shortcut, notes
+):
+ import time
+ from concurrent.futures import ThreadPoolExecutor
+
+ from fastapi.testclient import TestClient
+
+ from robots.dual_franka.robot_spec import DUAL_FRANKA_DASHBOARD_SPEC
+ from rpent.dashboard.operator import DashboardOperator
+ from rpent.dashboard.server import DashboardServer
+ from rpent.dashboard.state import DashboardState
+
+ t, env, _ = build(confirm=False)
+ state = DashboardState(
+ output_dir=tmp_path / "web", dashboard_spec=DUAL_FRANKA_DASHBOARD_SPEC
+ )
+ state.shared_services_ready()
+ state.request_task({"task_id": 3})
+ state.wait_for_task(0)
+ state.operator = DashboardOperator(tmp_path / "web")
+ t._operator_input = state.operator
+ state.bind_toolkit(t)
+ with (
+ TestClient(DashboardServer(state=state)._app) as client,
+ ThreadPoolExecutor() as pool,
+ ):
+ future = pool.submit(call, t, "request_scene_reset", reason="web reset")
+ try:
+ deadline = time.monotonic() + 3
+ while state.operator.snapshot() is None and time.monotonic() < deadline:
+ time.sleep(0.01)
+ response = client.get("/api/session/state")
+ assert response.status_code == 200
+ data = response.json()["operator"]
+ assert data["enabled"] and data["active"]
+ assert data["pending"]["choices"] == ["done", "abort"]
+ assert env.resets == 0
+ message = {
+ "text": "/done"
+ if shortcut and reply == "done"
+ else f"/operator {data['pending']['id']} {reply}",
+ "operator_context": {
+ "generation": data["generation"],
+ "request_id": data["pending"]["id"],
+ },
+ }
+ if notes:
+ message["text"] += f" {notes}"
+ invalid_messages = [
+ (
+ {"text": message["text"]},
+ "No active operator channel",
+ ),
+ (
+ {**message, "text": "/operator"},
+ "Usage: /operator",
+ ),
+ (
+ {**message, "operator_context": {"generation": 99}},
+ "expired operator task",
+ ),
+ (
+ {**message, "text": f"/operator stale-request {reply}"},
+ "No matching pending operator request",
+ ),
+ (
+ {**message, "text": f"/operator {data['pending']['id']} success"},
+ "invalid operator response",
+ ),
+ ]
+ invalid_messages.extend(
+ (
+ {**message, "operator_context": invalid_context},
+ "operator_context must be an object",
+ )
+ for invalid_context in ([], "invalid", 123, True)
+ )
+ if shortcut and reply == "done":
+ invalid_messages.append(
+ (
+ {
+ **message,
+ "operator_context": {
+ **message["operator_context"],
+ "request_id": "stale-request",
+ },
+ },
+ "No matching pending operator request",
+ )
+ )
+ for invalid_message, error in invalid_messages:
+ rejected = client.post("/api/session/messages", json=invalid_message)
+ assert rejected.status_code == 422
+ assert error in rejected.json()["error"]
+ assert not future.done() and env.resets == 0
+ assert (
+ client.get("/api/session/state").json()["operator"]["pending"]
+ == data["pending"]
+ )
+ assert client.post("/api/session/messages", json=message).status_code == 202
+ result = future.result(timeout=3)
+ assert env.resets == (1 if reply == "done" else 0)
+ if reply == "abort":
+ assert result["operator_aborted"]
+ finish = call(t, "finish", status="failure", summary="operator abort")
+ assert finish["_finish"] and finish["operator_aborted"]
+ assert not t.solved()
+ assert (
+ client.get("/api/session/state").json()["operator"]["pending"] is None
+ )
+ duplicate = client.post("/api/session/messages", json=message)
+ assert duplicate.status_code == 422
+ assert "No matching pending operator request" in duplicate.json()["error"]
+ assert env.resets == (1 if reply == "done" else 0)
+ finally:
+ state.operator.close()
+ state.unbind_toolkit(t)
+
+
+@pytest.mark.parametrize("verdict", ["success", "failure", "abort"])
+def test_dashboard_terminal_verdict_closes_only_current_task(build, tmp_path, verdict):
+ from fastapi.testclient import TestClient
+
+ from robots.dual_franka.robot_spec import DUAL_FRANKA_DASHBOARD_SPEC
+ from rpent.dashboard.operator import DashboardOperator
+ from rpent.dashboard.server import DashboardServer
+ from rpent.dashboard.state import DashboardState
+
+ t, _, _ = build()
+ state = DashboardState(
+ output_dir=tmp_path / "web", dashboard_spec=DUAL_FRANKA_DASHBOARD_SPEC
+ )
+ state.shared_services_ready()
+ state.request_task({"task_id": 3})
+ state.wait_for_task(0)
+ state.operator = DashboardOperator(tmp_path / "web")
+ state.bind_toolkit(t)
+ with TestClient(DashboardServer(state=state)._app) as client:
+ message = {
+ "text": f"/{verdict} test",
+ "operator_context": {"generation": 1},
+ }
+ assert (
+ client.post(
+ "/api/session/messages",
+ json=message,
+ ).status_code
+ == 202
+ )
+ assert t.direct_verdict_requested
+ assert "motion_refused" in call(t, "open_gripper", arm="right")
+ t.finalize_direct_verdict()
+ assert t.solved() is (verdict == "success")
+ duplicate = client.post("/api/session/messages", json=message)
+ assert duplicate.status_code == 422
+ assert "no matching active operator task" in duplicate.json()["error"]
+ assert client.get("/api/session/state").json()["operator"]["verdict"] == verdict
+ state.unbind_toolkit(t)
+ state.complete_task(state="cancelled")
+ assert (
+ state._task_state
+ == {"success": "succeeded", "failure": "failed", "abort": "cancelled"}[
+ verdict
+ ]
+ )
+ state.request_task({"task_id": 3})
+ state.wait_for_task(0)
+ assert state.operator_verdict is None
+ next_toolkit, _, _ = build()
+ state.operator = DashboardOperator(tmp_path / "next-web")
+ state.bind_toolkit(next_toolkit)
+ try:
+ late = client.post("/api/session/messages", json=message)
+ assert late.status_code == 422
+ assert "expired operator task" in late.json()["error"]
+ assert not next_toolkit.direct_verdict_requested
+ snapshot = client.get("/api/session/state").json()["operator"]
+ assert snapshot["generation"] == 2
+ assert snapshot["verdict"] is None
+ finally:
+ state.unbind_toolkit(next_toolkit)
+
+
+def test_dashboard_cancel_releases_pending_operator_wait(build, tmp_path):
+ import time
+ from concurrent.futures import ThreadPoolExecutor
+
+ from rpent.dashboard.operator import DashboardOperator
+
+ t, env, _ = build(confirm=False)
+ t._operator_input = DashboardOperator(tmp_path)
+ with ThreadPoolExecutor() as pool:
+ future = pool.submit(call, t, "request_scene_reset", reason="wait")
+ deadline = time.monotonic() + 3
+ while t._operator_input.snapshot() is None and time.monotonic() < deadline:
+ time.sleep(0.01)
+ t.cancel_active_and_wait()
+ future.result(timeout=3)
+ assert env.resets == 0
+ assert t._operator_input.snapshot() is None
+
+
+@pytest.mark.parametrize(
+ "verdict,auto_merge,planner_error,replace_task",
+ [
+ ("success", True, None, False),
+ ("success", False, None, False),
+ ("success", True, "planner failed", False),
+ ("failure", True, None, False),
+ ("abort", True, None, False),
+ (None, True, None, True),
+ ("success", True, None, True),
+ ],
+)
+@pytest.mark.parametrize("explore", [False, True])
+def test_dashboard_task_orchestration_uses_web_operator(
+ tmp_path,
+ monkeypatch,
+ verdict,
+ auto_merge,
+ planner_error,
+ replace_task,
+ explore,
+ dual_franka_robot_config,
+):
+ import time
+ from concurrent.futures import ThreadPoolExecutor
+ from dataclasses import replace
+
+ from robots.dual_franka import robot_spec
+ from rpent.cli import dashboard, main
+ from rpent.dashboard.state import DashboardState
+ from rpent.planner.base import PlannerResult
+
+ parser = main._build_argparser()
+ robot_spec._add_cli_args(parser, use_dashboard=True)
+ args = parser.parse_args(
+ [
+ "--robot",
+ "dual_franka",
+ "--dashboard",
+ "--task-id",
+ "3",
+ "--robot-config",
+ str(dual_franka_robot_config),
+ "--planner",
+ "codex",
+ "--memory-profile",
+ "local",
+ "--memory-dir",
+ str(tmp_path / "memory"),
+ "--auto-merge-memory" if auto_merge else "--no-auto-merge-memory",
+ ]
+ )
+ env = Env()
+
+ args.explore = explore
+
+ def runtime(_task_args, *unused):
+ assert env.resets == 0
+ return [], {
+ "env": env,
+ "model": None,
+ "task_description": "test",
+ }
+
+ spec = replace(robot_spec.get_robot_spec(), init_runtime=runtime)
+ state = DashboardState(output_dir=tmp_path, dashboard_spec=spec.dashboard)
+ state.shared_services_ready()
+ state.request_task({"task_id": 3})
+ claimed = state.wait_for_task(0)
+ merge_calls = []
+ merge_memory = MemoryManager.merge_memory
+
+ def merge(manager, **kwargs):
+ merge_calls.append(kwargs)
+ return merge_memory(manager, **kwargs)
+
+ monkeypatch.setattr(MemoryManager, "merge_memory", merge)
+
+ class Planner:
+ def solve(self, **kwargs):
+ t = kwargs["toolkit"]
+ assert kwargs["dashboard_interaction"] is state
+ assert t._attended and env.resets == 0
+ assert "error" in call(t, "move_delta", arm="right", delta_xyz=[0, 0, 0.01])
+ with ThreadPoolExecutor() as pool:
+ future = pool.submit(
+ call, t, "request_scene_reset", reason="fake web test"
+ )
+ try:
+ deadline = time.monotonic() + 3
+ while (
+ state.operator.snapshot() is None
+ and time.monotonic() < deadline
+ ):
+ time.sleep(0.01)
+ data = state.operator_snapshot()
+ state.operator_reply(
+ data["generation"], data["pending"]["id"], "done"
+ )
+ future.result(3)
+ finally:
+ if not future.done():
+ state.operator.close()
+ assert env.resets == 1
+ if verdict is not None:
+ state.operator_finish(1, verdict, "fake operator verdict")
+ if replace_task:
+ state.request_task({"task_id": 4})
+ return PlannerResult(messages=[], stats={}, error=planner_error)
+
+ monkeypatch.setattr(dashboard, "build_planner", lambda *a, **kw: Planner())
+ error = dashboard._run_dashboard_task(
+ args=args,
+ robot_spec=spec,
+ state=state,
+ claimed=claimed,
+ shared_runtime_kwargs={},
+ unique_components={"env"},
+ session_root=tmp_path,
+ )
+ assert error == planner_error
+ assert env.resets == 1
+ if explore and verdict in {"success", "failure"}:
+ evidence = list((tmp_path / "memory").rglob(f"*-{verdict}.json"))
+ assert len(evidence) == 1
+ assert (
+ json.loads(evidence[0].read_text())["verdict"]["operator_verdict"]
+ == verdict
+ )
+ should_merge = (
+ explore
+ and verdict == "success"
+ and auto_merge
+ and planner_error is None
+ and not replace_task
+ )
+ assert len(merge_calls) == int(should_merge)
+ for name in ("dual_franka_t3.json", "dual_franka_t3_recipe.jsonl"):
+ assert (tmp_path / "memory" / "task-specific" / name).is_file() is should_merge
+ state.complete_task(state="cancelled", error=error)
+ assert state._task_state == (
+ "cancelled"
+ if error
+ else {
+ "success": "succeeded",
+ "failure": "failed",
+ "abort": "cancelled",
+ None: "cancelled",
+ }[verdict]
+ )
+ if replace_task:
+ replacement = state.wait_for_task(0)
+ assert replacement is not None
+ assert replacement.number == 2
+ assert replacement.request == {"task_id": 4}
diff --git a/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py b/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py
index 72ffa752b..a1af36642 100644
--- a/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py
+++ b/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py
@@ -18,9 +18,11 @@
from pathlib import Path
+import pytest
+
from robots.dual_franka import get_robot_spec
from robots.dual_franka.prompt_bundle import system_prompt, user_prompt
-from robots.dual_franka.runtime_config import load_runtime_config
+from robots.dual_franka.runtime_config import load_mapping, load_runtime_config
from robots.dual_franka.tasks import CLEAN_DESK_VLA_PROMPT, DUAL_FRANKA_TASKS
from rpent.prompt.utils import format_prompt
from rpent.robots.base import enumerate_robots
@@ -43,20 +45,46 @@ def test_franka_extensions_declare_real_robot():
assert single.is_real_robot and dual.is_real_robot
+def test_manual_schemas_preserve_native_tool_declarations():
+ from robots.dual_franka.dual_franka_manual_call import (
+ _registered_tool_names,
+ _schema_payload,
+ )
+ from robots.dual_franka.toolkit import DualFrankaToolkit
+
+ tools = DualFrankaToolkit.declared_tools()
+ assert _registered_tool_names() == {tool.name for tool in tools}
+ for tool in tools:
+ assert _schema_payload(tool.name) == {
+ "primitive": tool.name,
+ "description": tool.description,
+ "input_schema": tool.input_schema,
+ }
+ assert _schema_payload("reset")["description"].startswith("Manual-only")
+
+
def test_dual_franka_uses_rpent_owned_robot_config(fake_rlinf_realworld_modules):
config_path = Path(__file__).parents[4] / "robots/dual_franka/config/example.yaml"
runtime = load_runtime_config(None, task_description="test task")
cfg = runtime.rlinf
assert config_path.is_file()
- assert cfg.env.eval.init_params.id == "DualFrankaTCPEnv-v1"
+ assert cfg.env.eval.init_params.id == "RPentDualFrankaTCPEnv-v1"
assert cfg.env.eval.override_cfg.task_description == "test task"
assert runtime.controller["move_tolerance_m"] == 0.006
assert runtime.controller["rotate_tolerance_rad"] == 0.04
+ assert runtime.controller["rotate_position_tolerance_m"] == 0.006
+ assert runtime.controller["rotate_max_drift_m"] == 0.03
+ assert runtime.controller["rotate_integral_gain_per_s"] == 0.5
+ assert runtime.controller["rotate_integral_limit_rad"] == 0.12
+ assert runtime.controller["rotate_position_integral_gain_per_s"] == 0.5
+ assert runtime.controller["rotate_position_integral_limit_m"] == 0.015
assert runtime.controller["iteration_multiplier"] == 50
assert runtime.controller["min_iterations"] == 200
assert runtime.controller["gripper_settle_s"] == 1.5
- assert runtime.controller["joint_health_thresholds"]["left"]["warning_q1"] == 1.7
+ thresholds = runtime.controller["joint_health_thresholds"]
+ assert thresholds["left"] == thresholds["right"]
+ assert thresholds == load_mapping(config_path)["joint_health"]["thresholds"]
def test_clean_desk_task_registers_named_vla_skills_and_fixed_prompt():
@@ -66,7 +94,7 @@ def test_clean_desk_task_registers_named_vla_skills_and_fixed_prompt():
assert "bowls, plates, cup, chopsticks, then spoon" in task.instruction
assert "vla_right_grasp" in "\n".join(task.constraints)
assert "segment" in "\n".join(task.constraints)
- assert "recover_joint_posture" in "\n".join(task.constraints)
+ assert "joint_health" not in "\n".join(task.constraints)
assert CLEAN_DESK_VLA_PROMPT.startswith(
"I am currently performing a desk organizing task."
)
@@ -95,6 +123,42 @@ def test_dirty_clean_exploration_candidate_reuses_deployed_task_prompt():
assert "dirty bowls/plates to the metal wire basket/frame" in candidate_constraints
+def test_two_cup_sorting_task_extends_exploration_candidate():
+ from robots.dual_franka.tasks import get_dual_franka_task
+
+ task = get_dual_franka_task(6)
+ original = DUAL_FRANKA_TASKS[4]
+ prompt = "\n".join(
+ (task.instruction, task.setup, task.success_criteria, *task.constraints)
+ )
+ assert task.name.endswith("two_cups_explore_candidate")
+ assert task.vla_instruction == original.vla_instruction
+ assert "exactly two cups" in task.instruction
+ assert "egg-tart foil tray" in task.setup
+ assert "Chicken nuggets inside a bowl" in task.setup
+ assert "bowl marker is not an egg tart or foil tray" in task.setup
+ assert "Chicken nuggets inside a bowl" in "\n".join(task.constraints)
+ assert "dirty bowls, plates, and cups go to the metal basket" in prompt
+ assert "clean bowls, plates, and cups go to the cardboard box" in prompt
+ assert "Process bowls -> plates -> both cups." in prompt
+ assert "There are no chopsticks or spoons to process" in task.instruction
+ assert "chopstick" not in "\n".join(task.constraints)
+ assert "spoon" not in task.success_criteria
+ assert "If no clean cup is available" not in prompt
+ assert "Both cups are accounted for separately" in task.success_criteria
+ assert "the cup still goes" not in prompt.lower()
+ assert "clean bowls/plates and the cup" not in prompt
+ assert "10cm above" not in prompt
+ for constraint in (*original.constraints[:2], original.constraints[19]):
+ assert constraint in task.constraints
+ for index in (8, 10, 12, 14, 18, 20):
+ assert original.constraints[index] not in task.constraints
+ assert "Contact is not restricted to VLA" in prompt
+ assert "does not require stopping for operator feedback" in prompt
+ assert "arm/object motion can improve visibility" in prompt
+ assert "The cup still goes into the cardboard box" in original.instruction
+
+
def test_metal_basket_task_registers_non_sorting_prompt():
task = DUAL_FRANKA_TASKS[5]
constraints = "\n".join(task.constraints)
@@ -105,19 +169,176 @@ def test_metal_basket_task_registers_non_sorting_prompt():
assert "Every object must go through one right-to-left handoff" in (
task.instruction
)
- assert "the cup does not need to support later utensil insertion" in (
- task.instruction
- )
+ assert "The spoon must be inserted into the cup" in task.instruction
+ assert "no other destination is acceptable" in task.instruction
+ assert "outside the cup does not satisfy the task" in task.success_criteria
assert "Do not perform dirty/clean classification" in constraints
assert "The metal wire basket/frame is the only valid placement container" in (
constraints
)
- assert "The utensil destination is the metal basket" in constraints
+ assert (
+ "Insert the spoon into the cup; no other destination is acceptable"
+ in constraints
+ )
assert "a failed grasp or drop is recoverable" in constraints
assert "Treat the two chopsticks as two separate objects" in task.instruction
assert "inspect the right_wrist artifact as primary evidence" in constraints
assert "bowls -> plates -> cup -> chopsticks -> spoon" in constraints
assert "green object before the blue object" in constraints
+ assert "inside_metal_basket without in_cup is a wrong destination" in constraints
+ assert "Track the spoon as complete only when inserted into the cup" in constraints
+
+
+def test_task5_placement_does_not_prescribe_alignment():
+ task = DUAL_FRANKA_TASKS[5]
+ prompt = format_prompt(system_prompt())
+ combined = prompt + "\n" + "\n".join(task.constraints)
+ assert "Put bowls and plates into the metal basket" in combined
+ for directive in (
+ "delta_z=0",
+ "strict staging gate",
+ "latest staging move",
+ "free-space x/y alignment",
+ ):
+ assert directive not in combined
+ for task_id in (1, 3):
+ assert "keep the current left TCP z" in "\n".join(
+ DUAL_FRANKA_TASKS[task_id].constraints
+ )
+
+
+def test_task6_treats_residual_error_as_diagnostic():
+ from robots.dual_franka.prompts import explore, system
+
+ rules = "\n".join(system.RULES)
+ assert "target_reached=false alone is not a controller" in rules
+ assert "protective motion aborts" in rules
+ assert explore.base.RULES is system.RULES
+ constraints = "\n".join(DUAL_FRANKA_TASKS[6].constraints)
+ assert "proceed even when target_reached is false" in constraints
+ assert "Do not repeatedly micro-adjust" in constraints
+ assert "no fixed correction size or mandatory alignment" in constraints
+ assert "reports a safe/effective target" not in constraints
+ assert "does not reach the target, do not call" not in constraints
+
+
+def test_shared_prompt_does_not_prescribe_task_strategy():
+ for mode in ("eval", "explore"):
+ prompt = format_prompt(
+ system_prompt({"mode": mode}),
+ variables={
+ "output_dir": "/tmp/test-output",
+ "memory_inbox": "/tmp/test-memory",
+ "memory_dir": "/tmp/test-memory",
+ "session_number": 1,
+ "session_max": 1,
+ "recipe_tag": "dual_franka_t5",
+ "task_id": 5,
+ },
+ )
+ for phrase in (
+ "D455",
+ "vla_right_grasp",
+ "vla_handoff",
+ "vla_left_place",
+ "current category",
+ "delta_z=0",
+ "metal basket",
+ "active-vision",
+ "Use the task's VLA segment tools for contact-rich motion.",
+ "before manual pixel projection",
+ ):
+ assert phrase not in prompt
+
+
+@pytest.mark.parametrize("task_id", [0, 1, 3, 4, 5, 6])
+@pytest.mark.parametrize("explore", [False, True])
+def test_rendered_tasks_retain_their_own_operating_instructions(
+ task_id, explore, dual_franka_robot_config
+):
+ from rpent.cli.main import _build_argparser
+
+ spec = get_robot_spec()
+ parser = _build_argparser()
+ spec.add_cli_args(parser, use_dashboard=False)
+ argv = [
+ "--robot",
+ "dual_franka",
+ "--task-id",
+ str(task_id),
+ "--robot-config",
+ str(dual_franka_robot_config),
+ ]
+ if explore:
+ argv.append("--explore")
+ config = spec.parse_config(parser.parse_args(argv))
+ variables = {**config.prompt_vars, "output_dir": str(config.output_dir)}
+ task = DUAL_FRANKA_TASKS[task_id]
+ system = format_prompt(spec.prompts.system(variables), variables=variables)
+ user = format_prompt(spec.prompts.user(variables), variables=variables)
+ for phrase in ("joint_health", "joint-health", "recover_joint_posture"):
+ assert phrase not in system + user
+ for text in (
+ task.instruction,
+ task.setup,
+ task.success_criteria,
+ *task.constraints,
+ ):
+ assert text in user
+ if task_id == 0:
+ assert "Do not call a VLA skill" in user
+ assert "vla_handoff" not in system + user
+ return
+ for text in ("closure plus lift", "release delay", "left gripper", "only mean"):
+ assert text in user
+ assert "vla_handoff" in user and "vla_left_place" in user
+ if task_id == 5:
+ assert (
+ "this task imposes no small-motion requirement or fixed adjustment sequence"
+ in user
+ )
+ assert "Missing depth or ambiguous placement alone is not a reason" in user
+ assert "stop for operator feedback instead of opening" not in user
+ elif task_id != 6:
+ assert "Do not move" in user and "camera visibility" in user
+ if task_id in (1, 3, 5):
+ assert "localization table" in user or "localization and sorting table" in user
+ assert "source/table height" in user
+ if task_id in (1, 3):
+ assert "delta_z=0" in user
+ else:
+ assert "delta_z=0" not in user + system
+ if task_id == 4:
+ assert "localization and sorting table" not in user
+ assert "source/table height" not in user
+ if task_id == 5:
+ assert "left gripper actually holds the intended object" in user
+ assert "latest staging move" not in user + system
+ assert (
+ "For spoon placement, the cup interior is the required projection target"
+ in user
+ )
+ assert (
+ "Insert the spoon into the cup; no other destination is acceptable" in user
+ )
+ assert "Put the chopsticks into the metal basket" in user
+ assert "bowls -> plates -> cup -> chopsticks -> spoon" in user
+ assert "Choose available camera views and perception tools as useful" in user
+ assert "RGB-based visual control is allowed" in user
+ assert "does not require a successful depth projection first" in user
+ assert (
+ "Do not present RGB estimates or invalid depth as calibrated metric measurements"
+ in user
+ )
+ assert "Invalid depth is not a valid metric measurement" in user
+ for stale_rule in (
+ "the cup as a utensil destination",
+ "use segment or back_project again instead of",
+ "before falling back to a manual pixel",
+ "If it is unclear whether the object has lifted or remains on the table, call back_project",
+ "the marker is not on the intended target, retry or stop",
+ ):
+ assert stale_rule not in user
def test_dual_franka_exploration_prompt_is_opt_in():
diff --git a/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py b/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py
index c250f12fa..a8b97a058 100644
--- a/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py
+++ b/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py
@@ -617,6 +617,7 @@ def test_segment_returns_mask_overlay_and_world_point(tmp_path: Path):
config.write_text(
"perception:\n"
" calibration:\n"
+ f" base_camera: {fixtures / 'third_to_right_base_calib_eye_on_base.yaml'}\n"
f" d455_camera: {fixtures / 'd455_to_right_base_eye_on_base.yaml'}\n"
" projection_views:\n"
" d455:\n"
@@ -641,6 +642,16 @@ def test_segment_returns_mask_overlay_and_world_point(tmp_path: Path):
sam3_client=FakeSam3Client(),
)
result = result_native.data
+ from robots.dual_franka.perception import _mask_to_camera_world
+
+ rejected = _mask_to_camera_world(
+ np.ones((8, 8), dtype=bool),
+ np.full((8, 8), 3.958),
+ camera="d455",
+ state=state,
+ step_idx=step,
+ min_valid=1,
+ )
finally:
set_robot_config_path(None)
@@ -655,6 +666,12 @@ def test_segment_returns_mask_overlay_and_world_point(tmp_path: Path):
assert result["segment_artifact"].startswith("d455_segment_")
assert result["overlay_artifact"].startswith("d455_segment_overlay_")
assert result_native.images and all(result_native.images)
+
+ assert rejected["point_xyz"] is None
+ assert rejected["valid_localization_pixels"] == 0
+ assert rejected["raw_depth_median_m"] == 3.958
+ assert rejected["mask_bbox_rc"] == [0, 0, 7, 7]
+ assert len(rejected["centroid_pixel"]) == 2
assert result["image_block_order"] == ["d455_segment_overlay"]
blocks = mcp_result(result_native)["content"]
diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py
index b6e14d0c7..f1868d214 100644
--- a/tests/unit_tests/robots/dual_franka/test_exploration.py
+++ b/tests/unit_tests/robots/dual_franka/test_exploration.py
@@ -25,6 +25,7 @@
from robots.dual_franka.toolkit import DualFrankaToolkit
from rpent.dashboard.events import NullDashboardEventSink
from rpent.memory import MemoryManager
+from rpent.tools import ToolResult
class FakeEnv:
@@ -171,10 +172,22 @@ def fail():
assert not env.moves
-def test_operator_abort_allows_finish_even_with_budget_and_never_succeeds(setup):
+@pytest.mark.parametrize(
+ "response",
+ [
+ "abort",
+ "abort calibration board still attached",
+ " ABORT operator stop ",
+ None,
+ ],
+)
+def test_operator_abort_allows_finish_even_with_budget_and_never_succeeds(
+ setup, response
+):
t, env, replies = setup
- replies.append("abort")
- call(t, "request_scene_reset", reason="initial")
+ replies.append(response)
+ reset_result = call(t, "request_scene_reset", reason="initial")
+ assert reset_result["operator_aborted"]
result = call(t, "finish", status="success", summary="stop")
assert result["_finish"] and result["operator_aborted"]
assert result["status"] == "failure" and not t.solved() and env.resets == 0
@@ -293,22 +306,20 @@ def test_cli_two_sessions_operator_feedback_and_memory_pipeline(
from types import SimpleNamespace
from rpent.cli import main as cli
+ from rpent.tools.human_in_the_loop import HumanInTheLoopInput
env = FakeEnv()
runtimes = []
planners = []
replies = iter(["done", "failure dropped", "done", "success lifted"])
- class Operator:
- def __init__(self, **kwargs):
- assert kwargs["interactive"] is False
-
- def __call__(self, prompt, cancelled):
+ class Operator(HumanInTheLoopInput):
+ def request(self, prompt, cancelled, *, kind=None):
cancelled()
return next(replies)
- def close(self):
- pass
+ def __call__(self, prompt, cancelled):
+ return self.request(prompt, cancelled)
class Planner:
def __init__(self, *args, **kwargs):
@@ -367,6 +378,11 @@ def init_runtime(*args):
spec = replace(robot_spec.get_robot_spec(), init_runtime=init_runtime)
monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec)
monkeypatch.setattr(cli, "build_planner", Planner)
+ monkeypatch.setattr(
+ cli,
+ "start_interactive_reader",
+ lambda inputs, **kwargs: inputs.put(kwargs["first_prompt_default"]),
+ )
monkeypatch.setattr("rpent.tools.human_in_the_loop.HumanInTheLoopInput", Operator)
monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True))
monkeypatch.setattr(
@@ -376,6 +392,8 @@ def init_runtime(*args):
"rpent",
"--robot",
"dual_franka",
+ "--planner",
+ "codex",
"--explore",
"--explore-sessions",
"2",
@@ -529,7 +547,7 @@ def active_motion(**kwargs):
worker.start()
assert entered.wait(2)
assert t.request_direct_verdict("success")
- assert t.request_direct_verdict("success") # idempotent, never another attempt
+ assert not t.request_direct_verdict("success") # no duplicate planner EOF
assert call(t, "open_gripper", arm="left")["motion_refused"]
release.set()
worker.join(2)
@@ -567,6 +585,8 @@ def test_direct_success_with_failed_observation_does_not_publish(setup):
@pytest.mark.parametrize("robot_name", ["dual_franka", "libero"])
@pytest.mark.parametrize("verdict", ["success", "failure", "abort"])
@pytest.mark.parametrize("planner_error", [None, "planner transport failed"])
+@pytest.mark.parametrize("session_count", [1, 2])
+@pytest.mark.parametrize("consume_cancel", [False, True])
def test_cli_direct_verdict_finalizes_and_merges_only_without_errors(
tmp_path,
monkeypatch,
@@ -574,6 +594,8 @@ def test_cli_direct_verdict_finalizes_and_merges_only_without_errors(
verdict,
planner_error,
robot_name,
+ session_count,
+ consume_cancel,
):
import sys
from dataclasses import replace
@@ -583,20 +605,39 @@ def test_cli_direct_verdict_finalizes_and_merges_only_without_errors(
env = FakeEnv()
handlers = {}
+ solve_calls = []
def reader(input_queue, **kwargs):
handlers["line"] = kwargs["line_handler"]
+ handlers["inputs"] = input_queue
input_queue.put("test task")
monkeypatch.setattr(cli, "start_interactive_reader", reader)
class Planner:
def solve(self, *, toolkit, input_queue, **kwargs):
+ import queue
+
+ with pytest.raises(queue.Empty):
+ input_queue.get(block=False)
+ if solve_calls:
+ assert "write a grounded outcome summary" in kwargs["user_message"]
+ assert "Focus on the later task stages" in kwargs["user_message"]
+ assert not toolkit._scene_ready
+ if verdict == "success":
+ assert (
+ tmp_path / "memory/task-specific/dual_franka_t0_recipe.jsonl"
+ ).is_file()
+ solve_calls.append(True)
+ assert not handlers["line"]("Focus on the later task stages")
toolkit._operator_input = lambda *args: "done"
call(toolkit, "request_scene_reset", reason="test")
call(toolkit, "move_delta", arm="right", delta_xyz=[0.01, 0, 0])
- assert handlers["line"]("/" + verdict)
- assert input_queue.get(timeout=1) is None
+ assert handlers["line"]("/" + verdict + " 建议采用混合控制")
+ assert handlers["line"]("/" + verdict + " 重复提交")
+ if consume_cancel:
+ assert input_queue.get(timeout=1) is None
+ assert handlers["inputs"].empty()
return SimpleNamespace(
finish_result=None, messages=[], stats={}, error=planner_error
)
@@ -632,6 +673,8 @@ def solve(self, *, toolkit, input_queue, **kwargs):
"codex",
"--explore",
"--interactive",
+ "--explore-sessions",
+ str(session_count),
"--output-dir",
str(tmp_path / "run"),
"--memory-dir",
@@ -640,7 +683,25 @@ def solve(self, *, toolkit, input_queue, **kwargs):
str(dual_franka_robot_config),
],
)
- assert cli.main() == (1 if planner_error else 0)
+ assert cli.main() == (1 if planner_error and verdict != "failure" else 0)
+ if verdict == "failure":
+ failures = list((tmp_path / "memory/_internal/inbox").rglob("*-failure.json"))
+ assert len(failures) == session_count
+ assert json.loads(failures[0].read_text())["verdict"]["status"] == "failure"
+ assert (
+ json.loads(failures[0].read_text())["verdict"]["operator_notes"]
+ == "建议采用混合控制"
+ )
+ assert len(solve_calls) == (
+ session_count
+ if verdict == "failure" or (verdict == "success" and not planner_error)
+ else 1
+ )
+ if verdict == "success" and not planner_error:
+ assert (
+ len(list((tmp_path / "run/sessions").glob("*/dual_franka_t0_recipe.jsonl")))
+ == session_count
+ )
assert (tmp_path / "memory/task-specific/dual_franka_t0.json").is_file() == (
verdict == "success" and planner_error is None
)
@@ -651,17 +712,435 @@ def solve(self, *, toolkit, input_queue, **kwargs):
(tmp_path / "run/sessions/session_001/operator_events.json").read_text()
)
assert events[-1]["status"] == ("failure" if verdict == "abort" else verdict)
- assert events[-1]["operator_finished"] is True
+ assert events[-1]["operator_finished"] is (verdict == "abort")
+
+
+@pytest.mark.parametrize("last_verdict", ["failure", "abort", "unfinished"])
+def test_success_memory_survives_a_later_unsuccessful_session(
+ tmp_path, monkeypatch, last_verdict, dual_franka_robot_config
+):
+ import sys
+ from dataclasses import replace
+ from types import SimpleNamespace
+
+ from rpent.cli import main as cli
+
+ env = FakeEnv()
+ handlers = {}
+ calls = []
+ waiting_tools = []
+
+ def reader(input_queue, **kwargs):
+ handlers["line"] = kwargs["line_handler"]
+ input_queue.put("test task")
+
+ class Planner:
+ def solve(self, *, toolkit, input_queue, **kwargs):
+ assert not toolkit._scene_ready
+ if calls:
+ assert (tmp_path / "memory/task-specific/dual_franka_t0.json").is_file()
+ verdict = "success" if not calls else last_verdict
+ calls.append(verdict)
+ if verdict == "unfinished":
+ import threading
+
+ ready = threading.Event()
+ monkeypatch.setattr(
+ "builtins.print", lambda *args, **kwargs: ready.set()
+ )
+ worker = threading.Thread(
+ target=lambda: call(
+ toolkit, "request_scene_reset", reason="pending"
+ )
+ )
+ waiting_tools.append(worker)
+ worker.start()
+ assert ready.wait(2)
+ assert handlers["line"]("/abort")
+ return SimpleNamespace(
+ finish_result=None, messages=[], stats={}, error=None
+ )
+ toolkit._operator_input = lambda *args: "done"
+ call(toolkit, "request_scene_reset", reason="test")
+ call(toolkit, "move_delta", arm="right", delta_xyz=[0.01, 0, 0])
+ assert handlers["line"]("/" + verdict)
+ assert input_queue.get(timeout=1) is None
+ return SimpleNamespace(
+ finish_result=None, messages=[], stats={}, error=None
+ )
+
+ spec = replace(
+ robot_spec.get_robot_spec(),
+ init_runtime=lambda *args: (
+ [],
+ {"env": env, "model": None, "task_description": "test"},
+ ),
+ )
+ monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec)
+ monkeypatch.setattr(cli, "start_interactive_reader", reader)
+ monkeypatch.setattr(cli, "build_planner", lambda *args, **kwargs: Planner())
+ monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True))
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "rpent",
+ "--robot",
+ "dual_franka",
+ "--planner",
+ "codex",
+ "--explore",
+ "--interactive",
+ "--explore-sessions",
+ "50" if last_verdict == "unfinished" else "2",
+ "--output-dir",
+ str(tmp_path / "run"),
+ "--memory-dir",
+ str(tmp_path / "memory"),
+ "--robot-config",
+ str(dual_franka_robot_config),
+ ],
+ )
+ assert cli.main() == 0
+ assert calls == ["success", last_verdict]
+ published = json.loads(
+ (tmp_path / "memory/task-specific/dual_franka_t0.json").read_text()
+ )
+ assert "session_001" in published["state_trace"]
+ assert not (
+ tmp_path / "run/sessions/session_002/dual_franka_t0_recipe.jsonl"
+ ).exists()
+ assert env.resets == (1 if last_verdict == "unfinished" else 2)
+ for worker in waiting_tools:
+ worker.join(2)
+ assert not worker.is_alive()
+
+
+@pytest.mark.parametrize("kind", ["reset", "verdict"])
+def test_close_retires_waiting_request_without_robot_motion(setup, monkeypatch, kind):
+ import threading
+
+ from rpent.tools.human_in_the_loop import HumanInTheLoopInput
+
+ t, env, replies = setup
+ if kind == "verdict":
+ reset(t, replies)
+ resets_before = env.resets
+ broker = HumanInTheLoopInput(interactive=True)
+ t._operator_input = broker
+ ready = threading.Event()
+ monkeypatch.setattr("builtins.print", lambda *args, **kwargs: ready.set())
+ result = []
+ tool = "request_scene_reset" if kind == "reset" else "request_operator_verdict"
+ kwargs = {"reason": "test cancellation"} if kind == "reset" else {}
+ worker = threading.Thread(target=lambda: result.append(call(t, tool, **kwargs)))
+ worker.start()
+ assert ready.wait(2)
+ old_id = broker._pending[0]
+ t.close()
+ worker.join(2)
+ assert not worker.is_alive()
+ assert broker.pending_kind is None
+ assert env.resets == resets_before and not env.moves
+ assert call(t, "request_scene_reset", reason="late call")["motion_refused"]
+ assert broker.route_line(f"/operator {old_id} done")
+ assert broker.pending_kind is None
+
+
+@pytest.mark.parametrize("pending_reset", [False, True])
+def test_attended_reply_resumes_same_toolkit_without_automatic_reset(
+ setup, monkeypatch, tmp_path, pending_reset
+):
+ import queue
+ import threading
+ from types import SimpleNamespace
+
+ from rpent.cli.main import _solve_attended
+ from rpent.tools.human_in_the_loop import HumanInTheLoopInput
+
+ t, env, replies = setup
+ if not pending_reset:
+ reset(t, replies)
+ broker = HumanInTheLoopInput(interactive=True)
+ t._operator_input = broker
+ inputs = queue.Queue()
+ first_reply = threading.Event()
+ second_reply = threading.Event()
+ prompt_ready = threading.Event()
+ monkeypatch.setattr("builtins.print", lambda *a, **kw: prompt_ready.set())
+ workers = []
+ calls = []
+ servers = []
+
+ async def reset_over_http(url):
+ import httpx
+ from mcp import ClientSession
+ from mcp.client.streamable_http import streamable_http_client
+
+ async with httpx.AsyncClient(trust_env=False) as client:
+ async with streamable_http_client(url, http_client=client) as (
+ read,
+ write,
+ _,
+ ):
+ async with ClientSession(read, write) as session:
+ await session.initialize()
+ result = await session.call_tool(
+ "request_scene_reset", {"reason": "test"}
+ )
+ assert not result.isError
+
+ class Planner:
+ def solve(self, *, toolkit, user_message, **kwargs):
+ assert toolkit is t
+ server = kwargs["mcp_server"]
+ servers.append(server)
+ assert server._thread.is_alive() and not server._server.should_exit
+ calls.append(user_message)
+ if len(calls) == 1:
+ if pending_reset:
+ import asyncio
+
+ worker = threading.Thread(
+ target=lambda: asyncio.run(reset_over_http(server.url))
+ )
+ worker.start()
+ workers.append(worker)
+ assert prompt_ready.wait(2)
+ first_reply.set()
+ return SimpleNamespace(
+ finish_result=None,
+ messages=[{"role": "assistant", "content": "Please confirm"}],
+ stats={},
+ error=None,
+ )
+ assert "SAME attempt" in user_message
+ assert t._scene_ready
+ assert env.resets == 1
+ t.request_direct_verdict("success", "test complete")
+ second_reply.set()
+ return SimpleNamespace(
+ finish_result=None, messages=[], stats={}, error=None
+ )
+
+ results = []
+ runner = threading.Thread(
+ target=lambda: results.append(
+ _solve_attended(
+ Planner(),
+ operator_input=broker,
+ state_output_dir=tmp_path / "session",
+ keep_mcp_alive=True,
+ toolkit=t,
+ input_queue=inputs,
+ system_prompt="test",
+ user_message="original task",
+ max_turns=10,
+ )
+ )
+ )
+ runner.start()
+ assert first_reply.wait(2)
+ assert not second_reply.wait(0.15)
+ if pending_reset:
+ assert broker.route_line("/done 保留前半段成果")
+ else:
+ inputs.put("场景已调整，请继续后半段")
+ runner.join(3)
+ for worker in workers:
+ worker.join(2)
+ assert not runner.is_alive() and len(results) == 1
+ assert len(calls) == 2 and env.resets == 1
+ assert len(list((tmp_path / "session").glob("dialogue_*.json"))) == 2
+ assert not env.moves
+ assert servers[0] is servers[1]
+ assert servers[0]._thread is None
+
+
+def test_attended_wait_eof_aborts_without_another_planner_call(setup, tmp_path):
+ import queue
+ from types import SimpleNamespace
+
+ from rpent.cli.main import _solve_attended
+ from rpent.tools.human_in_the_loop import HumanInTheLoopInput
+
+ t, env, _ = setup
+ inputs = queue.Queue()
+ inputs.put(None)
+ calls = []
+
+ class Planner:
+ def solve(self, **kwargs):
+ calls.append(True)
+ return SimpleNamespace(
+ finish_result=None, messages=[], stats={}, error=None
+ )
+
+ _solve_attended(
+ Planner(),
+ operator_input=HumanInTheLoopInput(interactive=True),
+ state_output_dir=tmp_path / "session",
+ toolkit=t,
+ input_queue=inputs,
+ user_message="task",
+ system_prompt="",
+ max_turns=10,
+ )
+ assert calls == [True]
+ assert t.finalize_direct_verdict()["operator_aborted"]
+ assert not env.resets and not env.moves
+
+
+@pytest.mark.parametrize("reply_style", ["shortcut", "request_id"])
+def test_attended_consumes_continue_once_without_duplicate_wait(
+ setup, tmp_path, monkeypatch, reply_style
+):
+ import queue
+ import threading
+ from types import SimpleNamespace
+
+ from rpent.cli.main import _solve_attended
+ from rpent.tools.human_in_the_loop import HumanInTheLoopInput
+
+ t, env, _ = setup
+ broker = HumanInTheLoopInput(interactive=True)
+ ready = threading.Event()
+ monkeypatch.setattr("builtins.print", lambda *a, **kw: ready.set())
+ calls = []
+ inputs = queue.Queue()
+ inputs.put(None)
+
+ class Planner:
+ def solve(self, **kwargs):
+ calls.append(kwargs["user_message"])
+ if len(calls) == 1:
+ replies = []
+ thread = threading.Thread(
+ target=lambda: replies.append(
+ broker.request("continue?", lambda: None, kind="verdict")
+ )
+ )
+ thread.start()
+ assert ready.wait(2)
+ command = (
+ "/continue"
+ if reply_style == "shortcut"
+ else f"/operator {broker._pending[0]} continue"
+ )
+ assert broker.route_line(command)
+ thread.join(2)
+ assert replies == ["continue"]
+ return SimpleNamespace(
+ finish_result=None, messages=[], stats={}, error=None
+ )
+
+ _solve_attended(
+ Planner(),
+ operator_input=broker,
+ state_output_dir=tmp_path / "session",
+ toolkit=t,
+ input_queue=inputs,
+ user_message="task",
+ system_prompt="",
+ max_turns=10,
+ )
+ assert len(calls) == 2
+ assert "already answered continue" in calls[1]
+ assert broker.continue_revision == 1
+ assert t.finalize_direct_verdict()["operator_aborted"]
+ assert not env.resets and not env.moves
+
+
+def test_current_view_refreshes_but_historical_view_does_not(setup, monkeypatch):
+ t, env, replies = setup
+ reset(t, replies)
+ old_step = t.state.latest_step
+ reads = []
+ published = []
+ monkeypatch.setattr(t, "_publish_step", published.append)
+ original = env.get_observation
+
+ def observe():
+ reads.append(True)
+ obs = original()
+ obs["d455_images"][:] = 77
+ return obs
+
+ env.get_observation = observe
+ observed = t.execute_tool("view_env_state", {})
+ assert len(reads) == 1 and t.state.latest_step == old_step + 1
+ assert observed.images == [t.state.load_bytes("d455.png")]
+ assert len(published) == 1
+ assert published[0].command == {"action": "observe_current"}
+ assert np.all(t.state.load("d455.png") == 77)
+ call(t, "view_env_state", step=old_step)
+ assert len(reads) == 1
+ assert (
+ "error" in t._current_perception(lambda **kw: ToolResult(), step=old_step).data
+ )
+ t._current_perception(lambda **kw: ToolResult(), step=t.state.latest_step)
+ assert len(reads) == 1 # Localization must not silently acquire another frame.
+ assert len(published) == 1
+ assert env.resets == 1 and not env.moves
+ call(t, "move_delta", arm="left", delta_xyz=[0.01, 0, 0])
+ assert len(reads) == 2 and t.state.latest_step == old_step + 2
+ assert len(published) == 2
+ assert published[-1].command["action"] == "move_delta"
+ assert len(env.moves) == 1 and env.moves[0][0] == "left"
+ np.testing.assert_allclose(env.moves[0][1], [0.01, 0, 0])
+
+
+def test_continue_captures_scene_after_operator_adjustment(setup, monkeypatch):
+ t, env, replies = setup
+ reset(t, replies)
+ original = env.get_observation
+ scene = [10]
+ reads = []
+ published = []
+ monkeypatch.setattr(t, "_publish_step", published.append)
+
+ def observe():
+ reads.append(scene[0])
+ obs = original()
+ obs["d455_images"][:] = scene[0]
+ return obs
+
+ env.get_observation = observe
+
+ def operator(*args):
+ scene[0] = 90
+ return "continue moved object"
+
+ t._operator_input = operator
+ observed = t.execute_tool("request_operator_verdict", {})
+ result = observed.data
+ assert result["status"] == "continue"
+ assert np.all(t.state.load("d455.png", step=result["evidence_step"]) == 10)
+ assert np.all(t.state.load("d455.png", step=result["observation_step"]) == 90)
+ assert observed.images == [
+ t.state.load_bytes("d455.png", step=result["observation_step"])
+ ]
+ assert result["observation_step"] == result["evidence_step"] + 1
+ assert reads == [10, 90]
+ assert [record.step_idx for record in published] == [
+ result["evidence_step"],
+ result["observation_step"],
+ ]
+ assert [record.command for record in published] == [
+ {"action": "observe_for_verdict"},
+ {"action": "observe_current"},
+ ]
+ assert env.resets == 1 and not env.moves
def test_direct_failure_ends_without_success_memory_even_before_reset(setup):
t, env, _ = setup
assert t.request_direct_verdict("failure")
- assert t.request_direct_verdict("failure")
+ assert not t.request_direct_verdict("failure")
assert not t.request_direct_verdict("success")
assert call(t, "request_scene_reset", reason="late reset")["motion_refused"]
result = t.finalize_direct_verdict()
- assert result["status"] == "failure" and result["operator_finished"]
+ assert result["status"] == "failure" and not result["operator_finished"]
assert not t.solved() and env.resets == 0
assert t.write_recipe("dual_franka_t0") is None
diff --git a/tests/unit_tests/robots/dual_franka/test_explore_observation.py b/tests/unit_tests/robots/dual_franka/test_explore_observation.py
index 6a158e18b..fb13e4b96 100644
--- a/tests/unit_tests/robots/dual_franka/test_explore_observation.py
+++ b/tests/unit_tests/robots/dual_franka/test_explore_observation.py
@@ -14,6 +14,7 @@
"""Operator evidence must refresh without invoking reset or step."""
+import queue
import sys
from types import ModuleType, SimpleNamespace
@@ -23,7 +24,8 @@
from robots.dual_franka.env_server import _create_worker_class
-def test_worker_reads_before_reset_and_refreshes_cached_frames(monkeypatch):
+@pytest.fixture
+def worker(monkeypatch):
for name, attrs in {
"rlinf.envs.real.env": {"RealWorldEnv": object},
"rlinf.robotics.parts.cameras": {
@@ -31,12 +33,16 @@ def test_worker_reads_before_reset_and_refreshes_cached_frames(monkeypatch):
"CameraInfo": object,
},
"rlinf.scheduler": {"Worker": object},
+ "robots.franka.rpent_env": {"realsense_color_intrinsics": lambda camera: {}},
}.items():
module = ModuleType(name)
module.__dict__.update(attrs)
monkeypatch.setitem(sys.modules, name, module)
cls = _create_worker_class()
- worker = cls.__new__(cls)
+ return cls.__new__(cls)
+
+
+def test_worker_reads_before_reset_and_refreshes_cached_frames(worker):
reads = []
def raw_observation():
@@ -64,3 +70,59 @@ def raw_observation():
assert np.all(first["main_images"] == 1)
assert np.all(second["main_images"] == 2)
assert first["main_images"].shape == (4, 4, 3)
+
+ def unavailable(timeout):
+ raise queue.Empty
+
+ del worker._capture_perception_camera_snapshot
+ worker._perception_cameras = {"d455": SimpleNamespace(get_observation=unavailable)}
+ with pytest.raises(RuntimeError, match="refusing cached RGBD"):
+ worker.get_observation()
+
+
+@pytest.mark.parametrize("with_depth", [False, True])
+def test_perception_uses_official_rgbd_reading_without_rescaling_depth(
+ worker, with_depth
+):
+ frame = np.full((4, 6, 3), [10, 20, 30], dtype=np.uint8)
+ depth = np.full((4, 6), 0.42, dtype=np.float32)
+ calls = []
+
+ def observation(*, timeout):
+ calls.append(timeout)
+ return {"frame": frame, **({"depth": depth} if with_depth else {})}
+
+ info = SimpleNamespace(
+ camera_type="realsense", serial_number="mock-extra", enable_depth=with_depth
+ )
+ worker._perception_cameras = {
+ "extra": SimpleNamespace(
+ get_observation=observation,
+ get_frame=lambda **kwargs: pytest.fail("use the official RGB-D interface"),
+ camera_info=info,
+ depth_scale=0.001,
+ )
+ }
+ worker._perception_camera_meta = {}
+ result = worker._capture_perception_camera_snapshot()
+
+ assert calls == [2]
+ np.testing.assert_array_equal(result["raw_frames"]["extra_rgb"], frame[..., ::-1])
+ assert result["raw_frames"]["extra_rgb"].dtype == np.uint8
+ meta = result["camera_meta"]["extra_rgb"]
+ assert meta["rgb_shape"] == [4, 6, 3]
+ assert meta["serial_number"] == "mock-extra"
+ assert meta["depth_available"] is with_depth
+ assert meta["depth_enabled"] is with_depth
+ assert worker._perception_camera_meta["extra_rgb"] == meta
+ if with_depth:
+ assert meta["depth_shape"] == [4, 6]
+ np.testing.assert_array_equal(result["raw_depths"]["extra_rgb"], depth)
+ assert result["raw_depths"]["extra_rgb"].dtype == np.float32
+ depth[:] = 9
+ np.testing.assert_allclose(result["raw_depths"]["extra_rgb"], 0.42)
+ else:
+ assert meta["depth_shape"] is None
+ assert result["raw_depths"] == {}
+ frame[:] = 0
+ assert result["raw_frames"]["extra_rgb"][0, 0].tolist() == [30, 20, 10]
diff --git a/tests/unit_tests/robots/dual_franka/test_full_vla.py b/tests/unit_tests/robots/dual_franka/test_full_vla.py
new file mode 100644
index 000000000..24c054ee0
--- /dev/null
+++ b/tests/unit_tests/robots/dual_franka/test_full_vla.py
@@ -0,0 +1,145 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Continuous manual VLA cancellation tests without hardware or network."""
+
+import threading
+
+import numpy as np
+import pytest
+
+from tests.manual.dual_franka_full_vla import MANUAL_SKILLS, run_full_vla
+from tests.unit_tests.robots.dual_franka.test_vla_test import make_session
+
+
+def run(session, stop, events):
+ return run_full_vla(
+ session.env,
+ session.model,
+ prompt=session.prompt,
+ workspace=session.workspace,
+ expected_steps=session.expected_steps,
+ stopped=stop,
+ emit=events.append,
+ )
+
+
+def test_continues_past_twenty_chunks_without_reset_or_semantic_stops(tmp_path):
+ session, calls, _ = make_session(tmp_path)
+ count = [0]
+ stopped = threading.Event()
+
+ def step(actions):
+ assert actions.shape == (1, 20)
+ count[0] += 1
+ if count[0] == 501:
+ stopped.set()
+ return {"ok": True}
+
+ session.env.chunk_step = step
+ events = []
+ result = run(session, stopped.is_set, events)
+ assert count[0] == 501
+ assert result["chunks"] == 26
+ assert calls.count("predict") == 26
+ assert result["reason"] == "operator_stop"
+ assert events[-1]["steps"] == 501
+
+
+@pytest.mark.parametrize("stage", ["before_start", "observation", "prediction"])
+def test_stop_before_action_never_sends_prediction(tmp_path, stage):
+ session, calls, _ = make_session(tmp_path)
+ stop = threading.Event()
+ if stage == "before_start":
+ stop.set()
+ else:
+ owner, name = (
+ (session.env, "get_observation")
+ if stage == "observation"
+ else (session.model, "predict")
+ )
+ original = getattr(owner, name)
+
+ def interrupted(*args, **kwargs):
+ result = original(*args, **kwargs)
+ stop.set()
+ return result
+
+ setattr(owner, name, interrupted)
+ result = run(session, stop.is_set, [])
+ assert "execute" not in calls
+ assert result["steps"] == 0
+
+
+def test_external_stop_file_discards_remaining_chunk(tmp_path):
+ session, _, _ = make_session(tmp_path)
+ stop_file = tmp_path / "STOP"
+ calls = []
+
+ def step(actions):
+ calls.append(actions)
+ stop_file.touch()
+ return {"ok": True}
+
+ session.env.chunk_step = step
+ result = run(session, stop_file.exists, [])
+ assert len(calls) == result["steps"] == 1
+
+
+@pytest.mark.parametrize("result", [{"ok": False}, {"error": "controller fault"}])
+def test_reported_error_stops_without_retry(tmp_path, result):
+ session, _, _ = make_session(tmp_path)
+ calls = []
+ session.env.chunk_step = lambda actions: calls.append(actions) or result
+ events = []
+ with pytest.raises(RuntimeError, match="environment action failed"):
+ run(session, lambda: False, events)
+ assert len(calls) == 1
+ assert events[-1]["status"] == "failed"
+
+
+def test_rpc_exception_stops_without_retry(tmp_path):
+ session, calls, _ = make_session(tmp_path, fail=True)
+ with pytest.raises(RuntimeError, match="RPC timeout"):
+ run(session, lambda: False, [])
+ assert calls.count("execute") == 1
+
+
+def test_invalid_prediction_never_moves(tmp_path):
+ session, calls, _ = make_session(tmp_path)
+ session.model.predict = lambda *args, **kwargs: np.full((20, 20), np.nan)
+ with pytest.raises(ValueError, match="finite actions"):
+ run(session, lambda: False, [])
+ assert "execute" not in calls
+
+
+@pytest.mark.parametrize("field", ["terminated", "truncated"])
+def test_environment_stop_is_not_automatically_reset(tmp_path, field):
+ session, _, _ = make_session(tmp_path)
+ calls = []
+ session.env.chunk_step = lambda actions: calls.append(actions) or {field: True}
+ result = run(session, lambda: False, [])
+ assert len(calls) == 1
+ assert result["status"] == "environment_stopped"
+
+
+def test_manual_registry_is_not_agent_schema():
+ from robots.dual_franka.toolkit import DualFrankaToolkit
+ from robots.dual_franka.tools import DualFrankaPrimitives
+
+ assert "vla_full_rollout" in MANUAL_SKILLS
+ assert "vla_full_rollout" not in {
+ tool.name for tool in DualFrankaToolkit.declared_tools()
+ }
+ assert not hasattr(DualFrankaPrimitives, "vla_full_rollout")
diff --git a/tests/unit_tests/robots/dual_franka/test_review_regressions.py b/tests/unit_tests/robots/dual_franka/test_review_regressions.py
index fd20ca810..f7b7cfbde 100644
--- a/tests/unit_tests/robots/dual_franka/test_review_regressions.py
+++ b/tests/unit_tests/robots/dual_franka/test_review_regressions.py
@@ -50,6 +50,144 @@ def worker_classes(monkeypatch):
return single(), dual()
+def _robot_with_parts(parts):
+ return SimpleNamespace(
+ child=lambda side: SimpleNamespace(child=parts[side].__getitem__)
+ )
+
+
+@pytest.mark.parametrize("arm", ["left", "right"])
+@pytest.mark.parametrize("tracking", [True, False])
+@pytest.mark.parametrize("timeout", [False, True])
+def test_move_completion_is_independent_of_accuracy(
+ worker_classes, monkeypatch, arm, tracking, timeout
+):
+ from robots.dual_franka import env_server
+
+ worker = worker_classes[1].__new__(worker_classes[1])
+ worker.per_arm_dim = 10
+ worker.controller = {
+ "move_max_step_m": 0.03,
+ "move_timeout_s": 1,
+ "move_tolerance_m": 0.006,
+ "min_iterations": 2,
+ "iteration_multiplier": 1,
+ }
+ clock = [0.0]
+ monkeypatch.setattr(env_server, "time", SimpleNamespace(time=lambda: clock[0]))
+ poses = np.array([[0.5, 0, 0.4, 0, 0, 0, 1.0]] * 2)
+ index = ["left", "right"].index(arm)
+ worker._refresh_robot_state = lambda: None
+ worker._arm_poses = lambda: (poses[0].copy(), poses[1].copy())
+ worker._pose_to_world = lambda arm, p: p.copy()
+ worker._pose_from_world = lambda arm, p: p.copy()
+
+ def step(action, *, auto_reset):
+ assert not auto_reset
+ if tracking:
+ poses[index, :3] = action[0, index * 10 : index * 10 + 3]
+ if timeout:
+ clock[0] = 2.0
+
+ worker.env = SimpleNamespace(step=step)
+ result = worker.move_delta(arm, [0.02, 0, 0])
+ assert result["ok"]
+ assert result["status"] == "completed"
+ assert result["message"] == "Action completed."
+ assert result["target_reached"] == tracking
+ assert result["final_error_m"] == pytest.approx(0 if tracking else 0.02, abs=1e-6)
+
+ def failing_step(*args, **kwargs):
+ raise RuntimeError("controller failed")
+
+ worker.env.step = failing_step
+ with pytest.raises(RuntimeError, match="controller failed"):
+ worker.move_delta(arm, [0.02, 0, 0])
+
+
+def test_health_scores_preserve_native_robot_errors(worker_classes):
+ cls = worker_classes[1]
+ worker = cls.__new__(cls)
+ worker._refresh_robot_state = lambda: None
+ state = {
+ "arm_joint_position": [0.0] * 7,
+ "has_errors": True,
+ "current_errors": {"joint_position_limits_violation": True},
+ "joint_collision": [True] + [False] * 6,
+ }
+ worker._arm_states = lambda: (dict(state), dict(state))
+ worker.action_dim = 20
+ worker.per_arm_dim = 10
+ worker.action_scale = np.ones(3)
+ worker.controller = {"joint_health_thresholds": {"left": {}, "right": {}}}
+ worker._raw_rlinf_env = lambda: SimpleNamespace(
+ config=SimpleNamespace(joint_reset_qpos=[[0.0] * 7, [0.0] * 7], is_dummy=True)
+ )
+ result = worker.get_robot_state()
+ assert result["joint_health"]["left"]["status"] == "critical"
+ assert "joint_collision" in result["joint_health"]["left"]["reasons"]
+ for arm in ("left_arm", "right_arm"):
+ assert result[arm]["has_errors"] is True
+ assert result[arm]["current_errors"] == state["current_errors"]
+ assert result[arm]["joint_collision"] == state["joint_collision"]
+
+
+def test_missing_native_diagnostics_are_unknown(worker_classes):
+ native = worker_classes[1]._native_joint_health({})
+ assert native["critical_reasons"] == []
+ assert native["warning_reasons"] == []
+ assert native["summary"]["available"] is False
+ assert native["summary"]["status"] == "unknown"
+ for key in (
+ "has_errors",
+ "current_errors",
+ "last_motion_errors",
+ "joint_contact",
+ "cartesian_contact",
+ "joint_collision",
+ "cartesian_collision",
+ ):
+ assert native["summary"][key] is None
+
+
+def test_present_empty_native_diagnostics_are_not_missing(worker_classes):
+ state = {
+ "robot_mode": "Move",
+ "has_errors": False,
+ "current_errors": {},
+ "last_motion_errors": {},
+ "joint_contact": [False] * 7,
+ "cartesian_contact": [False] * 6,
+ "joint_collision": [False] * 7,
+ "cartesian_collision": [False] * 6,
+ }
+ summary = worker_classes[1]._native_joint_health(state)["summary"]
+ assert summary["available"] is True
+ assert summary["status"] == "ok"
+ assert summary["current_errors"] == []
+ assert summary["joint_collision"] is False
+
+
+def test_joint_health_respects_configured_margin_thresholds(worker_classes):
+ cls = worker_classes[1]
+ worker = cls.__new__(cls)
+ worker.controller = {
+ "joint_health_thresholds": {
+ "left": {"warning_min_joint_margin": 0.2, "critical_min_joint_margin": 0.1}
+ }
+ }
+ q = [0.0, 0.0, 0.0, -1.5, 0.0, 2.0, 0.0]
+ worker._raw_rlinf_env = lambda: SimpleNamespace(
+ config=SimpleNamespace(joint_reset_qpos=[q.copy(), q.copy()])
+ )
+ state = {"arm_joint_position": q}
+ assert worker._joint_health_for_arm("left", state)["status"] == "ok"
+ q[-1] = 2.8973 - 0.15
+ assert worker._joint_health_for_arm("left", state)["status"] == "warning"
+ q[-1] = 2.8973 - 0.05
+ assert worker._joint_health_for_arm("left", state)["status"] == "critical"
+
+
@pytest.mark.parametrize("dual", [False, True])
def test_last_rotation_step_recomputes_error(worker_classes, dual):
cls = worker_classes[int(dual)]
@@ -60,6 +198,13 @@ def test_last_rotation_step_recomputes_error(worker_classes, dual):
"rotate_timeout_s": 10,
"rotate_tolerance_rad": 0.001,
"rotate_max_step_rad": 1,
+ "rotate_position_tolerance_m": 0.006,
+ "rotate_max_drift_m": 0.03,
+ "rotate_integral_gain_per_s": 0.0,
+ "rotate_integral_limit_rad": 0.12,
+ "rotate_position_integral_gain_per_s": 0.0,
+ "rotate_position_integral_limit_m": 0.015,
+ "rotate_settle_s": 0,
"min_iterations": 1,
"iteration_multiplier": 1,
}
@@ -90,6 +235,167 @@ def step(*args, **kwargs):
assert result["final_error_rad"] < 1e-6
+@pytest.mark.parametrize("arm", ["left", "right"])
+@pytest.mark.parametrize(
+ "drift,iterations,expected",
+ [
+ (0.012, 4, "reached"),
+ (0.012, 1, "iteration_limit"),
+ (0.04, 4, "position_drift"),
+ (0.04, 1, "position_drift"),
+ ],
+)
+def test_rotation_holds_initial_position_and_detects_drift(
+ worker_classes, arm, drift, iterations, expected
+):
+ worker = worker_classes[1].__new__(worker_classes[1])
+ worker.per_arm_dim = 10
+ worker.controller = {
+ "rotate_timeout_s": 10,
+ "rotate_tolerance_rad": 0.001,
+ "rotate_max_step_rad": 1,
+ "rotate_position_tolerance_m": 0.006,
+ "rotate_max_drift_m": 0.03,
+ "rotate_integral_gain_per_s": 0.0,
+ "rotate_integral_limit_rad": 0.12,
+ "rotate_position_integral_gain_per_s": 0.0,
+ "rotate_position_integral_limit_m": 0.015,
+ "rotate_settle_s": 0,
+ "min_iterations": iterations,
+ "iteration_multiplier": 1,
+ }
+ poses = np.array([[0.5, -0.2, 0.4, 0, 0, 0, 1.0], [0.4, 0.1, 0.5, 0, 0, 0, 1.0]])
+ initial = poses.copy()
+ index = ["left", "right"].index(arm)
+ frame_rotation = Rotation.from_euler("z", 0.4)
+ offset = np.array([0.01, 0.69, 0.0])
+
+ def to_world(name, pose):
+ p = pose.copy()
+ if name == "left":
+ p[:3] = frame_rotation.apply(p[:3]) + offset
+ p[3:] = (frame_rotation * Rotation.from_quat(p[3:])).as_quat()
+ return p
+
+ def from_world(name, pose):
+ p = pose.copy()
+ if name == "left":
+ p[:3] = frame_rotation.inv().apply(p[:3] - offset)
+ p[3:] = (frame_rotation.inv() * Rotation.from_quat(p[3:])).as_quat()
+ return p
+
+ worker._pose_to_world = to_world
+ worker._pose_from_world = from_world
+ worker._refresh_robot_state = lambda: None
+ worker._arm_poses = lambda: (poses[0].copy(), poses[1].copy())
+ initial_action = worker._hold_action(*initial)
+ start = to_world(arm, initial[index])
+ target = start.copy()
+ target[3:] = (
+ Rotation.from_euler("z", 0.6) * Rotation.from_quat(start[3:])
+ ).as_quat()
+ target_local = from_world(arm, target)
+ commands = []
+
+ def step(action, *, auto_reset):
+ assert not auto_reset
+ command = action[0]
+ commands.append(command.copy())
+ base = index * 10
+ np.testing.assert_allclose(
+ command[base : base + 3], initial[index, :3], atol=1e-6
+ )
+ np.testing.assert_array_equal(command[[9, 19]], [0, 0])
+ other = (1 - index) * 10
+ np.testing.assert_array_equal(
+ command[other : other + 10], initial_action[other : other + 10]
+ )
+ poses[index] = target_local
+ if len(commands) == 1:
+ poses[index, 1] += drift
+
+ worker.env = SimpleNamespace(step=step)
+ result = worker.rotate_delta(arm, [0, 0, 0.6])
+ assert result["exit_reason"] == expected
+ assert result["ok"] == (expected != "position_drift")
+ assert result["target_reached"] == (expected == "reached")
+ assert result["status"] == (
+ "aborted" if expected == "position_drift" else "completed"
+ )
+ assert result["rotation_reached"]
+ assert result["max_position_error_m"] == pytest.approx(drift)
+ assert len(commands) == (2 if expected == "reached" else 1)
+ if expected == "reached":
+ assert result["final_position_error_m"] < 1e-6
+ else:
+ assert not result["position_reached"]
+
+
+@pytest.mark.parametrize("gain", [0.0, 0.5])
+@pytest.mark.parametrize("angle", [0.1, 0.6])
+@pytest.mark.parametrize("tracking_bias", [0.06, 0.12, 0.3])
+@pytest.mark.parametrize("position_bias", [0.0, 0.01])
+def test_rotation_compensates_static_tracking_error(
+ worker_classes, monkeypatch, gain, angle, tracking_bias, position_bias
+):
+ from robots.dual_franka import env_server
+
+ worker = worker_classes[1].__new__(worker_classes[1])
+ worker.per_arm_dim = 10
+ worker.controller = {
+ "rotate_timeout_s": 20,
+ "rotate_tolerance_rad": 0.04,
+ "rotate_max_step_rad": 0.1,
+ "rotate_position_tolerance_m": 0.006,
+ "rotate_max_drift_m": 0.03,
+ "rotate_integral_gain_per_s": gain,
+ "rotate_position_integral_gain_per_s": gain,
+ "rotate_position_integral_limit_m": 0.015,
+ "rotate_settle_s": 0.5,
+ "rotate_integral_limit_rad": 0.12,
+ "min_iterations": 200,
+ "iteration_multiplier": 1,
+ }
+ ticks = iter(np.arange(0, 100, 0.1))
+ monkeypatch.setattr(
+ env_server,
+ "time",
+ SimpleNamespace(time=lambda: 0.0, monotonic=lambda: next(ticks)),
+ )
+ pose = np.array([0.5, 0, 0.4, 0, 0, 0, 1.0])
+ worker._refresh_robot_state = lambda: None
+ worker._arm_poses = lambda: (pose.copy(), pose.copy())
+ worker._pose_to_world = lambda arm, p: p.copy()
+ worker._pose_from_world = lambda arm, p: p.copy()
+ commands = []
+
+ def step(action, **kwargs):
+ r6 = action[0, 3:9]
+ matrix = np.column_stack([r6[:3], r6[3:], np.cross(r6[:3], r6[3:])])
+ reference = Rotation.from_matrix(matrix)
+ commands.append(reference)
+ pose[:3] = action[0, :3] + [0, position_bias, 0]
+ pose[3:] = (Rotation.from_euler("z", -tracking_bias) * reference).as_quat()
+
+ worker.env = SimpleNamespace(step=step)
+ result = worker.rotate_delta("left", [0, 0, angle])
+ expected_success = bool(gain) and tracking_bias < 0.15
+ assert result["ok"]
+ assert result["status"] == "completed"
+ assert result["target_reached"] == expected_success
+ assert np.linalg.norm(result["position_integral"]) <= 0.015001
+ assert np.linalg.norm(result["integral_rotvec"]) <= 0.120001
+ if angle == 0.6:
+ assert commands[1].as_rotvec()[2] == pytest.approx(0.2)
+ for previous, current in zip(commands, commands[1:]):
+ assert (current * previous.inv()).magnitude() <= 0.100001
+ if expected_success:
+ assert result["final_error_rad"] <= 0.04
+ assert result["final_position_error_m"] <= 0.006
+ else:
+ assert result["exit_reason"] == "iteration_limit"
+
+
@pytest.mark.parametrize("opened,expected", [(True, 1), (False, -1)])
def test_single_arm_motion_preserves_gripper(worker_classes, opened, expected):
worker = worker_classes[0].__new__(worker_classes[0])
@@ -121,30 +427,259 @@ def test_joint_reset_waits_for_both_controller_results(worker_classes):
def controller(arm):
def reset_joint(qpos):
- def wait():
- waited.append((arm, qpos))
- return [None]
-
- return SimpleNamespace(wait=wait)
+ waited.append((arm, qpos))
+ return None
return SimpleNamespace(reset_joint=reset_joint)
raw = SimpleNamespace(
- _left_ctrl=controller("left"), _right_ctrl=controller("right")
+ robot=_robot_with_parts(
+ {arm: {"arm": controller(arm)} for arm in ("left", "right")}
+ )
)
worker._raw_rlinf_env = lambda: raw
assert worker._reset_both_joints_no_gripper([[1], [2]]) == {
- "left": [None],
- "right": [None],
+ "left": None,
+ "right": None,
}
assert sorted(waited) == [("left", [1]), ("right", [2])]
-def test_missing_gripper_state_is_not_silently_defaulted(worker_classes):
+def test_missing_hand_state_is_not_silently_defaulted(worker_classes):
worker = worker_classes[1].__new__(worker_classes[1])
- worker._arm_states = lambda: (SimpleNamespace(), SimpleNamespace())
- with pytest.raises(AttributeError, match="gripper_open"):
- worker._current_gripper_commands()
+ raw = SimpleNamespace(
+ config=SimpleNamespace(is_dummy=False),
+ robot=_robot_with_parts(
+ {arm: {"end_effector": SimpleNamespace()} for arm in ("left", "right")}
+ ),
+ )
+ worker._raw_rlinf_env = lambda: raw
+ with pytest.raises(AttributeError, match="is_open"):
+ worker._gripper_states()
+
+
+def test_refresh_uses_official_tcp_accessor(worker_classes):
+ worker = worker_classes[1].__new__(worker_classes[1])
+ calls = []
+ raw = SimpleNamespace(
+ config=SimpleNamespace(is_dummy=False),
+ get_tcp_pose=lambda: calls.append("read"),
+ )
+ worker._raw_rlinf_env = lambda: raw
+ worker._refresh_robot_state()
+ assert calls == ["read"]
+ raw.config.is_dummy = True
+ worker._refresh_robot_state()
+ assert calls == ["read"]
+
+
+def test_robot_state_reads_hands_without_mutating_arm_snapshots(worker_classes):
+ worker = worker_classes[1].__new__(worker_classes[1])
+ arm_state = {
+ "arm_joint_position": [0.0] * 7,
+ "gripper_open": False,
+ "gripper_position": 0.0,
+ }
+ hands = {
+ "left": SimpleNamespace(is_open=True, position=0.08),
+ "right": SimpleNamespace(is_open=False, position=0.025),
+ }
+ raw = SimpleNamespace(
+ config=SimpleNamespace(is_dummy=False, joint_reset_qpos=[[0.0] * 7, [0.0] * 7]),
+ get_tcp_pose=lambda: None,
+ robot=_robot_with_parts(
+ {arm: {"end_effector": hand} for arm, hand in hands.items()}
+ ),
+ )
+ worker._raw_rlinf_env = lambda: raw
+ worker._arm_states = lambda: (arm_state, arm_state)
+ worker.controller = {"joint_health_thresholds": {"left": {}, "right": {}}}
+ worker.action_dim = 20
+ worker.per_arm_dim = 10
+ worker.action_scale = np.ones(3)
+
+ result = worker.get_robot_state()
+ assert result["left_arm"]["gripper_open"] is True
+ assert result["left_arm"]["gripper_position"] == 0.08
+ assert result["right_arm"]["gripper_open"] is False
+ assert result["right_arm"]["gripper_position"] == 0.025
+ assert arm_state["gripper_open"] is False
+ assert arm_state["gripper_position"] == 0.0
+ hands["left"].is_open = False
+ hands["left"].position = 0.01
+ assert worker.get_robot_state()["left_arm"]["gripper_open"] is False
+ assert worker.get_robot_state()["left_arm"]["gripper_position"] == 0.01
+
+
+def test_hold_action_never_reissues_gripper_state(worker_classes):
+ worker = worker_classes[1].__new__(worker_classes[1])
+ left = np.array([0.5, 0.2, 0.4, 0, 0, 0, 1.0])
+ right = np.array([0.4, -0.2, 0.3, 0, 0, 0, 1.0])
+ action = worker._hold_action(left, right)
+ assert action.shape == (20,)
+ np.testing.assert_array_equal(action[[9, 19]], [0, 0])
+ np.testing.assert_allclose(action[:3], left[:3])
+ np.testing.assert_allclose(action[10:13], right[:3])
+ explicit = worker._hold_action(left, right, left_grip=-1, right_grip=1)
+ np.testing.assert_array_equal(explicit[[9, 19]], [-1, 1])
+
+
+@pytest.mark.parametrize("arm", ["left", "right"])
+@pytest.mark.parametrize("opened", [False, True])
+def test_set_gripper_commands_only_selected_hand(worker_classes, arm, opened):
+ worker = worker_classes[1].__new__(worker_classes[1])
+ worker.per_arm_dim = 10
+ worker.gripper_idx = 9
+ worker.controller = {
+ "gripper_timeout_s": 1,
+ "gripper_max_iterations": 2,
+ "gripper_settle_s": 0,
+ }
+ hands = {
+ name: SimpleNamespace(is_open=not opened, position=0.04)
+ for name in ("left", "right")
+ }
+ raw = SimpleNamespace(
+ config=SimpleNamespace(is_dummy=False),
+ get_tcp_pose=lambda: None,
+ robot=_robot_with_parts(
+ {name: {"end_effector": hand} for name, hand in hands.items()}
+ ),
+ )
+ worker._raw_rlinf_env = lambda: raw
+ pose = np.array([0.5, 0.2, 0.4, 0, 0, 0, 1.0])
+ worker._arm_poses = lambda: (pose.copy(), pose.copy())
+ worker.get_robot_state = worker._gripper_states
+ actions = []
+
+ def step(action, *, auto_reset):
+ assert auto_reset is False
+ actions.append(action)
+ hands[arm].is_open = opened
+
+ worker.env = SimpleNamespace(step=step)
+ result = worker.set_gripper(arm.upper(), open=opened)
+ assert result["ok"] is True
+ assert result["steps_used"] == 1
+ selected = ["left", "right"].index(arm)
+ assert actions[0][0, selected * 10 + 9] == (1 if opened else -1)
+ assert actions[0][0, (1 - selected) * 10 + 9] == 0
+ other = "right" if arm == "left" else "left"
+ assert hands[other].is_open == (not opened)
+
+
+@pytest.mark.parametrize("threshold", [0.0, -0.5, float("nan"), float("inf"), 0.5])
+def test_worker_requires_a_gripper_noop_band(worker_classes, monkeypatch, threshold):
+ from robots.dual_franka import env_server
+
+ events = []
+ module = types.ModuleType("robots.dual_franka.rpent_env")
+ module.register_rpent_dual_franka_env = lambda: events.append("register")
+ monkeypatch.setitem(sys.modules, module.__name__, module)
+ monkeypatch.setattr(runtime_config, "set_robot_config_path", lambda path: None)
+ monkeypatch.setattr(runtime_config, "validate_calibration_sources", lambda: None)
+ env_config = SimpleNamespace(
+ binary_gripper_threshold=threshold, action_scale=np.ones(3)
+ )
+ attrs = {
+ "PER_ARM_ACTION_DIM": 10,
+ "GRIPPER_IDX_IN_ARM": 9,
+ "config": env_config,
+ }
+
+ def make_env(*args, **kwargs):
+ events.append("construct")
+ return SimpleNamespace(
+ action_space=SimpleNamespace(shape=(20,)),
+ env=SimpleNamespace(call=lambda method, name: [attrs[name]]),
+ close=lambda: events.append("close"),
+ )
+
+ monkeypatch.setattr(sys.modules["rlinf.envs.real.env"], "RealWorldEnv", make_env)
+ cls = env_server._create_worker_class()
+ cls.worker_info = None
+ cls._open_perception_cameras = lambda self: None
+ cfg = SimpleNamespace(env=SimpleNamespace(eval={}))
+ if threshold == 0.5:
+ cls(cfg, {})
+ assert events == ["register", "construct"]
+ else:
+ with pytest.raises(
+ ValueError, match="binary_gripper_threshold must be positive"
+ ):
+ cls(cfg, {})
+ assert events == ["register", "construct", "close"]
+
+
+@pytest.mark.parametrize("left_open", [False, True])
+def test_joint_recovery_preserves_independent_hands(
+ worker_classes, monkeypatch, left_open
+):
+ from robots.dual_franka import env_server
+
+ monkeypatch.setattr(env_server.time, "sleep", lambda seconds: None)
+ worker = worker_classes[1].__new__(worker_classes[1])
+ expected = {"left": left_open, "right": not left_open}
+ calls = []
+ hands = {}
+ parts = {}
+ poses = np.array([[0.5, 0.0, 0.4, 0, 0, 0, 1.0]] * 2)
+ for arm in ("left", "right"):
+ hand = SimpleNamespace(is_open=expected[arm], position=0.02)
+
+ def command(arm, opened):
+ calls.append((arm, "open" if opened else "close"))
+ hands[arm].is_open = opened
+
+ def reset_joint(qpos, arm=arm):
+ calls.append((arm, "reset_joint"))
+ assert hands[arm].is_open == expected[arm]
+
+ hand.open = lambda arm=arm: command(arm, True)
+ hand.close = lambda arm=arm: command(arm, False)
+ hands[arm] = hand
+ parts[arm] = {
+ "arm": SimpleNamespace(reset_joint=reset_joint),
+ "end_effector": hand,
+ }
+ raw = SimpleNamespace(
+ config=SimpleNamespace(is_dummy=False, joint_reset_qpos=[[0.0] * 7, [0.0] * 7]),
+ robot=_robot_with_parts(parts),
+ get_tcp_pose=lambda: None,
+ )
+ worker._raw_rlinf_env = lambda: raw
+ worker._arm_states = lambda: (
+ {
+ "tcp_pose": poses[0],
+ "arm_joint_position": [0.0] * 7,
+ "gripper_open": False,
+ },
+ {
+ "tcp_pose": poses[1],
+ "arm_joint_position": [0.0] * 7,
+ "gripper_open": False,
+ },
+ )
+ worker._arm_poses = lambda: (poses[0].copy(), poses[1].copy())
+ worker._pose_to_world = lambda arm, pose: pose.copy()
+ worker.action_dim = 20
+ worker.per_arm_dim = 10
+ worker.action_scale = np.ones(3)
+ worker.controller = {
+ "joint_health_thresholds": {"left": {}, "right": {}},
+ "gripper_settle_s": 0,
+ "move_tolerance_m": 0.01,
+ "rotate_tolerance_rad": 0.1,
+ }
+ result = worker.recover_joint_posture(return_to_start=False)
+ assert result["ok"] is True
+ assert result["start"]["gripper_open"] == expected
+ assert result["final"]["gripper_open"] == expected
+ assert result["final"]["gripper_preserved"] is True
+ for arm in ("left", "right"):
+ commands = [command for side, command in calls if side == arm]
+ hold = "open" if expected[arm] else "close"
+ assert commands == [hold, "reset_joint", hold, hold]
def test_exploration_candidate_keeps_policy_instruction():
@@ -218,7 +753,11 @@ def payload(obs):
@pytest.mark.parametrize("dedicated", [False, True])
-def test_live_environment_does_not_inherit_coding_profile(tmp_path, dedicated):
+@pytest.mark.parametrize("deployment_configured", [False, True])
+def test_live_environment_does_not_inherit_coding_profile(
+ tmp_path, dedicated, deployment_configured
+):
+ (tmp_path / "repo").mkdir()
script = (
Path(__file__).resolve().parents[4] / "robots/dual_franka/rpent_live_env.sh"
)
@@ -241,12 +780,37 @@ def test_live_environment_does_not_inherit_coding_profile(tmp_path, dedicated):
"RPENT_CODEX_BASE_URL": "https://robot.invalid",
}
)
+ if deployment_configured:
+ env.update(
+ {
+ "RPENT_CODEX_MODEL": "experiment-model",
+ "RPENT_REASONING_EFFORT": "ultra",
+ "PI05_CHECKPOINT_PATH": str(tmp_path / "checkpoint"),
+ "DUAL_FRANKA_REPO_ID": "experiment-stats",
+ "RPENT_ROBOT_CONFIG": str(tmp_path / "robot.yaml"),
+ }
+ )
command = """source "$1"
[[ "$CODEX_HOME" != "$HOME/coding" ]]
[[ -z "${OPENAI_API_KEY:-}" && -z "${OPENAI_BASE_URL:-}" ]]
[[ "${CODEX_API_KEY:-}" == "${RPENT_CODEX_API_KEY:-}" ]]
[[ "${CODEX_BASE_URL:-}" == "${RPENT_CODEX_BASE_URL:-}" ]]
-[[ "$CODEX_SERVICE_TIER" == fast && "$RPENT_REASONING_EFFORT" == medium ]]
+[[ "$CODEX_SERVICE_TIER" == fast ]]
+[[ "$CODEX_MODEL" == "$RPENT_CODEX_MODEL" ]]
+"""
+ if deployment_configured:
+ command += """
+[[ "$RPENT_CODEX_MODEL" == experiment-model ]]
+[[ "$RPENT_REASONING_EFFORT" == ultra ]]
+[[ "$PI05_CHECKPOINT_PATH" == "$HOME/checkpoint" ]]
+[[ "$DUAL_FRANKA_REPO_ID" == experiment-stats ]]
+[[ "$RPENT_ROBOT_CONFIG" == "$HOME/robot.yaml" ]]
+"""
+ else:
+ command += """
+[[ "$RPENT_CODEX_MODEL" == gpt-5.5 ]]
+[[ "$RPENT_REASONING_EFFORT" == medium ]]
+[[ -z "${PI05_CHECKPOINT_PATH+x}" && -z "${DUAL_FRANKA_REPO_ID+x}" ]]
"""
subprocess.run(
["bash", "-eu", "-c", command, "test", str(script)], env=env, check=True
diff --git a/tests/unit_tests/robots/dual_franka/test_rlinf_contract.py b/tests/unit_tests/robots/dual_franka/test_rlinf_contract.py
index 562668252..0babed8c0 100644
--- a/tests/unit_tests/robots/dual_franka/test_rlinf_contract.py
+++ b/tests/unit_tests/robots/dual_franka/test_rlinf_contract.py
@@ -17,6 +17,12 @@
from __future__ import annotations
import dataclasses
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
# Keys RPent builds into ``env.eval.override_cfg`` and the ``DualFranka``
# hardware config. Kept here (not a runtime constant) so this test doubles as
@@ -47,6 +53,7 @@
"left_controller_node_rank",
"right_controller_node_rank",
"node_rank",
+ "compliance",
}
@@ -68,12 +75,99 @@ def test_hardware_keys_are_valid_rlinf_fields(fake_rlinf_realworld_modules):
assert not unknown, f"hardware keys not in DualFrankaConfig: {unknown}"
-def test_runtime_config_sets_realtime_mode(fake_rlinf_realworld_modules):
- from robots.dual_franka.runtime_config import load_runtime_config
+@pytest.mark.parametrize("custom_config", [False, True])
+def test_runtime_config_preserves_explicit_overrides(
+ fake_rlinf_realworld_modules, tmp_path, custom_config
+):
+ from robots.dual_franka.runtime_config import (
+ DEFAULT_CONFIG,
+ load_mapping,
+ load_runtime_config,
+ )
- runtime = load_runtime_config(None, task_description="test task")
+ config = load_mapping(DEFAULT_CONFIG)
+ path = None
+ if custom_config:
+ config["robot"]["compliance"] = {
+ "nullspace_stiffness": 1.5,
+ "translational_clip": 0.02,
+ "rotational_clip": 0.12,
+ "max_step_rad": 0.08,
+ }
+ config["joint_health"]["thresholds"] = {
+ "left": {"warning_min_joint_margin": 0.2, "critical_min_joint_margin": 0.1},
+ "right": {
+ "warning_min_joint_margin": 0.3,
+ "critical_min_joint_margin": 0.15,
+ },
+ }
+ path = tmp_path / "robot.yaml"
+ path.write_text(yaml.safe_dump(config))
+
+ runtime = load_runtime_config(path, task_description="test task")
hardware = runtime.rlinf.cluster.node_groups[0].hardware.configs[0]
assert hardware["realtime_config"] == "ignore"
+ assert set(hardware) == (
+ _HARDWARE_KEYS if custom_config else _HARDWARE_KEYS - {"compliance"}
+ )
+ assert hardware["left_controller_node_rank"] == 0
+ assert hardware["right_controller_node_rank"] == 1
+ if custom_config:
+ assert hardware["compliance"] == config["robot"]["compliance"]
+ assert (
+ runtime.controller["joint_health_thresholds"]
+ == config["joint_health"]["thresholds"]
+ )
+
+
+def test_runtime_uses_rpent_adapter_without_preview_extension(
+ fake_rlinf_realworld_modules,
+):
+ from robots.dual_franka.runtime_config import load_runtime_config
+
+ runtime = load_runtime_config(None, task_description="test task")
+
+ assert runtime.rlinf.env.eval.init_params.id == "RPentDualFrankaTCPEnv-v1"
+ assert runtime.rlinf.env.eval.auto_reset is False
+ assert "camera_preview_dir" not in runtime.controller
+
+
+@pytest.mark.parametrize("checkout", [None, "/tmp/explicit-rlinf-checkout"])
+def test_live_environment_only_uses_explicit_rlinf_checkout(tmp_path, checkout):
+ repo_root = Path(__file__).resolve().parents[4]
+ env = os.environ.copy()
+ env.pop("RLINF_REPO_PATH", None)
+ env.update(
+ RPENT_REPO_ROOT=str(repo_root),
+ RPENT_CODEX_HOME=str(tmp_path / "codex"),
+ RPENT_LIVE_MEMORY_DIR=str(tmp_path / "memory"),
+ PYTHONPATH="/tmp/existing-python-path",
+ )
+ if checkout is not None:
+ env["RLINF_REPO_PATH"] = checkout
+
+ result = subprocess.run(
+ [
+ "bash",
+ "--noprofile",
+ "--norc",
+ "-c",
+ 'source "$1"; printf "%s\\n%s\\n" "${RLINF_REPO_PATH-unset}" "$PYTHONPATH"',
+ "--",
+ str(repo_root / "robots/dual_franka/rpent_live_env.sh"),
+ ],
+ env=env,
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+
+ rlinf_checkout, python_path = result.stdout.splitlines()
+ assert rlinf_checkout == (checkout or "unset")
+ expected_path = [str(repo_root), "/tmp/existing-python-path"]
+ if checkout is not None:
+ expected_path.insert(0, checkout)
+ assert python_path.split(":") == expected_path
def test_controller_carries_calibration_mapping_for_ray_worker(
diff --git a/tests/unit_tests/robots/dual_franka/test_rpent_env.py b/tests/unit_tests/robots/dual_franka/test_rpent_env.py
new file mode 100644
index 000000000..db4c058c0
--- /dev/null
+++ b/tests/unit_tests/robots/dual_franka/test_rpent_env.py
@@ -0,0 +1,319 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Offline observation tests without starting robot, camera or Ray resources."""
+
+from __future__ import annotations
+
+import importlib.util
+import logging
+import queue
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+
+import cv2
+import gymnasium as gym
+import numpy as np
+import pytest
+
+
+@pytest.fixture
+def adapter(fake_rlinf_realworld_modules, monkeypatch):
+ cameras = sys.modules["rlinf.robotics.parts.cameras"]
+ official_realsense = cameras.RealSenseCamera
+
+ @dataclass
+ class CameraInfo:
+ name: str
+ serial_number: str
+ camera_type: str = "realsense"
+ resolution: tuple[int, int] = (640, 480)
+ fps: int = 15
+ enable_depth: bool = False
+ crop_region: tuple[float, float, float, float] | None = None
+
+ class Camera:
+ backends = {"realsense": official_realsense}
+
+ @classmethod
+ def register(cls, name):
+ def add(driver):
+ assert name not in cls.backends
+ cls.backends[name] = driver
+ return driver
+
+ return add
+
+ class DualFrankaTCPEnv:
+ def __init__(self, **kwargs):
+ self.constructor_args = kwargs
+ self.closed = False
+
+ def _camera_infos(self):
+ return self.declarations
+
+ def _crop_frame(self, frame, size, interpolation=cv2.INTER_LINEAR):
+ height, width = frame.shape[:2]
+ side = min(height, width)
+ x, y = (width - side) // 2, (height - side) // 2
+ cropped = frame[y : y + side, x : x + side]
+ return cropped, cv2.resize(cropped, size, interpolation=interpolation)
+
+ def close(self):
+ self.closed = True
+
+ monkeypatch.setattr(cameras, "CameraInfo", CameraInfo)
+ monkeypatch.setattr(cameras, "Camera", Camera, raising=False)
+ monkeypatch.setattr(
+ sys.modules["rlinf.envs.real.franka.dual_franka_tcp"],
+ "DualFrankaTCPEnv",
+ DualFrankaTCPEnv,
+ raising=False,
+ )
+ path = Path(__file__).parents[4] / "robots/dual_franka/rpent_env.py"
+ spec = importlib.util.spec_from_file_location("_test_dual_franka_adapter", path)
+ module = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, spec.name, module)
+ spec.loader.exec_module(module)
+ return module
+
+
+def make_camera(adapter, name, readings, *, depth=False, realsense=False):
+ camera = adapter.RPentRealSenseCamera() if realsense else SimpleNamespace()
+ camera.camera_info = adapter.CameraInfo(
+ name,
+ f"serial-{name}",
+ camera_type="rpent_realsense" if realsense else "lumos",
+ resolution=(6, 4),
+ enable_depth=depth,
+ )
+ camera.depth_scale = 0.001 if depth else 1.0
+ camera.calls = []
+ observations = iter(readings)
+
+ def read(**kwargs):
+ camera.calls.append(kwargs)
+ result = next(observations)
+ if isinstance(result, Exception):
+ raise result
+ return result
+
+ camera.get_observation = read
+ if realsense:
+ camera.get_color_intrinsics = lambda: {
+ "width": 6,
+ "height": 4,
+ "fx": 5.0,
+ "fy": 5.0,
+ "ppx": 3.0,
+ "ppy": 2.0,
+ }
+ return camera
+
+
+def make_env(adapter, cameras):
+ env = adapter.RPentDualFrankaEnv()
+ env._cameras = cameras
+ env._last_camera_frame = {}
+ env._logger = logging.getLogger(__name__)
+ env.observation_space = {
+ "frames": {name: SimpleNamespace(shape=(2, 2, 3)) for name in cameras}
+ }
+ env.camera_player = SimpleNamespace(put_frame=lambda frames: None)
+ return env
+
+
+def test_camera_declarations_preserve_names_order_depth_and_official_backend(adapter):
+ env = adapter.RPentDualFrankaEnv()
+ original = [
+ adapter.CameraInfo("base_0_rgb", "base"),
+ adapter.CameraInfo("left_wrist_0_rgb", "left", camera_type="lumos"),
+ adapter.CameraInfo("right_wrist_0_rgb", "right", camera_type="lumos"),
+ ]
+ env.declarations = original
+ declarations = env._camera_infos()
+ assert [item.name for item in declarations] == [item.name for item in original]
+ assert [item.serial_number for item in declarations] == ["base", "left", "right"]
+ assert not any(item.enable_depth for item in declarations)
+ assert declarations[0].camera_type == "rpent_realsense"
+ assert original[0].camera_type == "realsense"
+ assert declarations[1:] == original[1:]
+ assert adapter.Camera.backends["realsense"] is adapter.RealSenseCamera
+
+
+def test_policy_and_raw_rgbd_share_one_acquisition_and_snapshot_owns_arrays(adapter):
+ bgr = np.arange(72, dtype=np.uint8).reshape(4, 6, 3)
+ depth = np.arange(24, dtype=np.float32).reshape(4, 6) / 100.0
+ camera = make_camera(
+ adapter,
+ "base_0_rgb",
+ [{"frame": bgr, "depth": depth}],
+ depth=True,
+ realsense=True,
+ )
+ env = make_env(adapter, {"base_0_rgb": camera})
+ frames, depths = env._get_camera_observation()
+ snapshot = env.get_raw_camera_snapshot()
+ np.testing.assert_array_equal(
+ frames["base_0_rgb"], cv2.resize(bgr[:, 1:5], (2, 2))[..., ::-1]
+ )
+ np.testing.assert_array_equal(
+ depths["base_0_rgb"],
+ cv2.resize(depth[:, 1:5], (2, 2), interpolation=cv2.INTER_NEAREST),
+ )
+ np.testing.assert_array_equal(snapshot["raw_frames"]["base_0_rgb"], bgr[..., ::-1])
+ np.testing.assert_array_equal(snapshot["raw_depths"]["base_0_rgb"], depth)
+ meta = snapshot["camera_meta"]["base_0_rgb"]
+ assert meta["depth_unit"] == "m"
+ assert meta["rgb_shape"] == [4, 6, 3]
+ assert meta["color_intrinsics"]["width"] == 6
+ assert meta["projection_supported"] is True
+ snapshot["raw_frames"]["base_0_rgb"][:] = 0
+ snapshot["raw_depths"]["base_0_rgb"][:] = 0
+ again = env.get_raw_camera_snapshot()
+ np.testing.assert_array_equal(again["raw_frames"]["base_0_rgb"], bgr[..., ::-1])
+ np.testing.assert_array_equal(again["raw_depths"]["base_0_rgb"], depth)
+ assert camera.calls == [{"timeout": 0.5, "max_age": 0.5}]
+
+
+def test_stalled_camera_keeps_policy_fallback_but_invalidates_raw_snapshot(adapter):
+ first = {"frame": np.full((4, 6, 3), 15, dtype=np.uint8)}
+ recovered = {"frame": np.full((4, 6, 3), 80, dtype=np.uint8)}
+ camera = make_camera(
+ adapter, "right_wrist_0_rgb", [first, queue.Empty(), recovered]
+ )
+ env = make_env(adapter, {"right_wrist_0_rgb": camera})
+ env._get_camera_observation()
+ assert env.get_raw_camera_snapshot()["raw_frames"]["right_wrist_0_rgb"].max() == 15
+ frames, _ = env._get_camera_observation()
+ assert frames["right_wrist_0_rgb"].max() == 15
+ with pytest.raises(RuntimeError, match="No fresh complete camera snapshot"):
+ env.get_raw_camera_snapshot()
+ with pytest.raises(RuntimeError, match="No fresh complete camera snapshot"):
+ env.get_raw_camera_metadata()
+ env._get_camera_observation()
+ assert env.get_raw_camera_snapshot()["raw_frames"]["right_wrist_0_rgb"].max() == 80
+
+
+def test_failed_first_acquisition_has_no_fallback_or_partial_snapshot(adapter):
+ valid = {"frame": np.zeros((4, 6, 3), dtype=np.uint8)}
+ env = make_env(
+ adapter,
+ {
+ "left_wrist_0_rgb": make_camera(adapter, "left_wrist_0_rgb", [valid]),
+ "right_wrist_0_rgb": make_camera(
+ adapter, "right_wrist_0_rgb", [queue.Empty()]
+ ),
+ },
+ )
+ with pytest.raises(RuntimeError, match="right_wrist_0_rgb stalled"):
+ env._get_camera_observation()
+ with pytest.raises(RuntimeError, match="No fresh complete camera snapshot"):
+ env.get_raw_camera_snapshot()
+
+
+def test_rgb_only_camera_does_not_claim_depth_or_projection_support(adapter):
+ env = make_env(
+ adapter,
+ {
+ "left_wrist_0_rgb": make_camera(
+ adapter,
+ "left_wrist_0_rgb",
+ [{"frame": np.zeros((4, 6, 3), dtype=np.uint8)}],
+ )
+ },
+ )
+ _, depths = env._get_camera_observation()
+ meta = env.get_raw_camera_metadata()["left_wrist_0_rgb"]
+ assert depths == {}
+ assert meta["camera_type"] == "lumos"
+ assert meta["color_intrinsics"] is None
+ assert meta["depth_available"] is False
+ assert meta["depth_unit"] is None
+ assert meta["projection_supported"] is False
+
+
+def test_realsense_metadata_reads_only_the_owned_profile(adapter, monkeypatch):
+ intrinsics = SimpleNamespace(
+ width=640,
+ height=480,
+ fx=600.0,
+ fy=601.0,
+ ppx=320.0,
+ ppy=240.0,
+ model="brown_conrady",
+ coeffs=[0.0] * 5,
+ )
+ profile = SimpleNamespace(get_intrinsics=lambda: intrinsics)
+ profile.as_video_stream_profile = lambda: profile
+ realsense = ModuleType("pyrealsense2")
+ realsense.stream = SimpleNamespace(color=object())
+ monkeypatch.setitem(sys.modules, "pyrealsense2", realsense)
+ camera = adapter.RPentRealSenseCamera()
+ camera.profile = SimpleNamespace(get_stream=lambda stream: profile)
+ metadata = camera.get_color_intrinsics()
+ assert metadata["fx"] == 600.0
+ assert metadata["fy"] == 601.0
+ assert metadata["width"] == 640
+ assert metadata["coeffs"] == [0.0] * 5
+
+
+def test_factory_uses_standard_wrapper_stack_and_closes_on_failure(
+ adapter, monkeypatch
+):
+ seen = []
+ wrapped = object()
+
+ def build(env, config):
+ seen.append((env, config))
+ return wrapped
+
+ monkeypatch.setattr(adapter, "build_stack", build)
+ kwargs = {
+ "override_cfg": {},
+ "worker_info": None,
+ "robot_info": None,
+ "env_idx": 0,
+ "env_cfg": {},
+ }
+ assert adapter.create_rpent_dual_franka_env(**kwargs) is wrapped
+ assert seen[0][0].constructor_args == {
+ key: value for key, value in kwargs.items() if key != "env_cfg"
+ }
+
+ def fail(env, config):
+ seen.append((env, config))
+ raise ValueError("invalid wrapper config")
+
+ monkeypatch.setattr(adapter, "build_stack", fail)
+ with pytest.raises(ValueError, match="invalid wrapper config"):
+ adapter.create_rpent_dual_franka_env(**kwargs)
+ assert seen[-1][0].closed
+
+
+def test_registration_is_idempotent_and_keeps_official_id(adapter, monkeypatch):
+ env_id = "RPentDualFrankaTCPEnv-v1"
+ monkeypatch.delitem(gym.registry, env_id, raising=False)
+ before = dict(gym.registry)
+ adapter.register_rpent_dual_franka_env()
+ adapter.register_rpent_dual_franka_env()
+ assert gym.spec(env_id).entry_point == (
+ "robots.dual_franka.rpent_env:create_rpent_dual_franka_env"
+ )
+ assert {
+ key: value for key, value in gym.registry.items() if key != env_id
+ } == before
+ monkeypatch.delitem(gym.registry, env_id)
diff --git a/tests/unit_tests/robots/dual_franka/test_vla_test.py b/tests/unit_tests/robots/dual_franka/test_vla_test.py
index 2e0192a85..6c806523f 100644
--- a/tests/unit_tests/robots/dual_franka/test_vla_test.py
+++ b/tests/unit_tests/robots/dual_franka/test_vla_test.py
@@ -156,8 +156,9 @@ def test_dual_client_uses_live_state_instead_of_cached_reset():
@pytest.mark.parametrize("instruction", [None, "diagnostic prompt"])
+@pytest.mark.parametrize("expected_steps", [None, 20])
def test_session_accepts_standard_config_without_local_deployment(
- tmp_path, monkeypatch, instruction
+ tmp_path, monkeypatch, instruction, expected_steps
):
import argparse
@@ -171,14 +172,22 @@ def test_session_accepts_standard_config_without_local_deployment(
closed = []
observed = []
args = argparse.Namespace(
- expected_action_steps=20,
+ expected_action_steps=expected_steps,
robot_config=str(config_path),
task_id=1,
instruction=instruction,
vla_model_path="checkpoint",
vla_repo_id="dataset",
)
- model = SimpleNamespace()
+ metadata = {"model_path": "checkpoint", "config": {"openpi": {"action_chunk": 7}}}
+ status_calls = []
+
+ def status(**kwargs):
+ assert expected_steps is None, "explicit chunk length must support old servers"
+ status_calls.append(kwargs)
+ return metadata
+
+ model = SimpleNamespace(status=status)
def init_runtime(args, output, events, components):
assert components == {"env", "vla"}
@@ -193,10 +202,19 @@ def init_runtime(args, output, events, components):
)
monkeypatch.setattr(vla_test, "get_robot_spec", lambda: spec)
monkeypatch.setattr(
- vla_test, "run_console", lambda session: observed.append(session.prompt)
+ vla_test, "run_console", lambda session: observed.append(session)
)
assert vla_test.run_session(args) == 0
- assert observed == [instruction or get_dual_franka_task(1).vla_instruction]
+ assert observed[0].prompt == (
+ instruction or get_dual_franka_task(1).vla_instruction
+ )
+ assert observed[0].expected_steps == (
+ 7 if expected_steps is None else expected_steps
+ )
+ deployment = json.loads((tmp_path / "run" / "deployment.json").read_text())
+ assert deployment["expected_action_steps"] == observed[0].expected_steps
+ assert deployment["server"] == (metadata if expected_steps is None else None)
+ assert status_calls == ([{"timeout_s": 10}] if expected_steps is None else [])
assert closed == [True]
@@ -317,3 +335,119 @@ def test_dual_franka_spawns_shared_vla_server():
assert command[command.index("--repo-id") + 1] == "test/data"
assert command[command.index("--model-path") + 1] == "/checkpoint"
assert command[command.index("--cuda-device") + 1] == "2"
+
+
+def test_vla_status_queries_metadata_without_inference():
+ from unittest.mock import Mock
+
+ from rpent.robots.components.pi05_vla_client import Pi05VLAClient
+
+ metadata = {"config": {"openpi": {"action_chunk": 20}}}
+ rpc = Mock()
+ rpc.call.return_value = metadata
+ client = Pi05VLAClient(rpc, embodiment="dual_franka")
+ assert client.status(timeout_s=5) == metadata
+ rpc.call.assert_called_once_with("vla.status", timeout_s=5)
+
+
+def test_status_command_keeps_explicit_validation_length(tmp_path):
+ session, calls, _ = make_session(tmp_path)
+ metadata = {"config": {"openpi": {"action_chunk": 7}}, "device": "cuda:0"}
+ session.model.status = lambda **kwargs: metadata
+ lines = iter(["status", "quit"])
+ output = []
+ run_console(session, read=lambda _: next(lines), emit=output.append)
+ result = json.loads(output[-1])
+ assert result["device"] == "cuda:0"
+ assert result["expected_action_steps"] == 20
+ assert result["prompt"] == "pick"
+ assert session.status == metadata
+ assert calls == []
+
+
+@pytest.mark.parametrize("reported_steps", [0, -1, None])
+def test_invalid_or_unavailable_status_does_not_fall_back(
+ tmp_path, monkeypatch, reported_steps
+):
+ from tests.e2e_tests.dual_franka import dual_franka_vla
+
+ config_path = tmp_path / "robot.yaml"
+ config_path.write_text("workspace: {}\n")
+ args = SimpleNamespace(
+ expected_action_steps=None,
+ robot_config=str(config_path),
+ task_id=1,
+ instruction="pick",
+ )
+ closed = []
+
+ def status(**kwargs):
+ if reported_steps is None:
+ raise RuntimeError("vla.status unavailable")
+ return {"config": {"openpi": {"action_chunk": reported_steps}}}
+
+ spec = SimpleNamespace(
+ parse_config=lambda args: SimpleNamespace(output_dir=tmp_path / "run"),
+ init_runtime=lambda *args: (
+ [SimpleNamespace(stop=lambda: closed.append(True))],
+ {"model": SimpleNamespace(status=status)},
+ ),
+ )
+ monkeypatch.setattr(dual_franka_vla, "get_robot_spec", lambda: spec)
+ monkeypatch.setattr(
+ dual_franka_vla,
+ "run_console",
+ lambda session: pytest.fail("invalid metadata must not reach the console"),
+ )
+ error = RuntimeError if reported_steps is None else ValueError
+ with pytest.raises(error, match="unavailable|must be positive"):
+ dual_franka_vla.run_session(args)
+ assert closed == [True]
+
+
+def test_dual_vla_status_and_prediction_follow_component_rpc_contract(monkeypatch):
+ import sys
+ from types import ModuleType
+
+ import torch
+
+ from rpent.robots.components.pi05_vla_server import Pi05VLAFacade
+
+ calls = []
+
+ class Model:
+ def cuda(self):
+ return self
+
+ def eval(self):
+ return self
+
+ def parameters(self):
+ return iter([torch.zeros(1)])
+
+ def predict_action_batch(self, obs, mode):
+ calls.append(mode)
+ return torch.ones((20, 20)), None
+
+ def get_model(cfg, torch_dtype):
+ assert cfg.action_dim == 20 and cfg.openpi.num_images_in_input == 3
+ assert cfg.openpi_data.repo_id == "test/dataset"
+ return Model()
+
+ loader = ModuleType("rlinf.models.embodiment.openpi")
+ loader.get_model = get_model
+ monkeypatch.setitem(sys.modules, loader.__name__, loader)
+ facade = Pi05VLAFacade(
+ model_path="/unused/checkpoint",
+ embodiment="dual_franka",
+ repo_id="test/dataset",
+ )
+ try:
+ status = facade._dispatch("vla.status", (), {})
+ assert status["config"]["openpi"]["action_chunk"] == 20
+ assert calls == []
+ actions = facade._dispatch("vla.predict", ({},), {"options": {"mode": "eval"}})
+ assert actions.shape == (20, 20) and actions.dtype == np.float32
+ assert calls == ["eval"]
+ finally:
+ facade.close()
diff --git a/tests/unit_tests/robots/fixtures/tool_schemas.json b/tests/unit_tests/robots/fixtures/tool_schemas.json
index 7a1bb3176..51d680643 100644
--- a/tests/unit_tests/robots/fixtures/tool_schemas.json
+++ b/tests/unit_tests/robots/fixtures/tool_schemas.json
@@ -1,6 +1,6 @@
{
"revision": "068cd64f4f175179a13c67d429a4d00ca56fb196",
- "source": "TOOLS_SPEC from the recorded revision; common file tools include MEMORY_BOUNDARY_NOTE appended by MemoryManager.get_common_tool_bindings().",
+ "source": "TOOLS_SPEC from the recorded revision; common file tools include MEMORY_BOUNDARY_NOTE appended by MemoryManager.get_common_tool_bindings(). Dual-Franka descriptions include the local observation, motion-completion and policy-capability contracts.",
"tools": {
"rpent/tools/common.py": [
{
@@ -940,7 +940,7 @@
},
{
"name": "view_env_state",
- "description": "Read a dual-Franka state snapshot. Configured inline camera views are returned directly; other available views are returned as artifact paths; use read_image to inspect these artifacts.",
+ "description": "Capture a fresh dual-Franka observation when step=-1 (default); a nonnegative step reads that historical snapshot without acquisition. Use the returned step for segmentation and back-projection; do not reuse pixels or masks from an older step after refreshing. Configured inline camera views are returned directly; other available views are returned as artifact paths for targeted read_image inspection.",
"input_schema": {
"type": "object",
"properties": {
@@ -992,7 +992,7 @@
},
{
"name": "move_delta",
- "description": "Move one Franka TCP by a bounded world-frame xyz delta in meters.",
+ "description": "Move one Franka TCP by a bounded world-frame xyz delta in meters. Completion indicates execution ended; target_reached and final_error_m report accuracy, not a mandatory retry condition.",
"input_schema": {
"type": "object",
"properties": {
@@ -1004,7 +1004,7 @@
},
{
"name": "rotate_delta",
- "description": "Rotate one Franka TCP by a bounded world-frame rpy delta in radians.",
+ "description": "Rotate one Franka TCP by a bounded world-frame rpy delta in radians. Completion indicates execution ended; target_reached and residual errors report accuracy, not a mandatory retry condition. Protective aborts remain failures.",
"input_schema": {
"type": "object",
"properties": {
@@ -1038,7 +1038,7 @@
},
{
"name": "recover_joint_posture",
- "description": "Reset both arms to their healthy configured joint posture while preserving each gripper's open/closed state. Closed grippers are re-commanded before/after the joint reset so held objects stay clamped, then both TCPs return near their prior poses.",
+ "description": "Reset both arms to their configured joint posture while preserving each gripper's open/closed state. Closed grippers are re-commanded before/after the joint reset so held objects stay clamped, then both TCPs return near their prior poses. This is an available recovery option, not a mandatory response to an advisory joint-health score.",
"input_schema": {
"type": "object",
"properties": {
@@ -1095,7 +1095,7 @@
},
{
"name": "vla_left_place",
- "description": "Run the learned left-placement VLA segment. The active task decides the destination; this tool only defines the capability boundary: left gripper opens and the left TCP lifts.",
+ "description": "Run the learned left-placement VLA segment under the configured fixed policy instruction. The prompt argument records planner intent; it does NOT select a destination or change policy conditioning. Use only when the policy's actual placement capability matches the intended destination. Boundary: left gripper opens and the left TCP lifts; this does not prove correct placement.",
"input_schema": {
"type": "object",
"properties": {
diff --git a/tests/unit_tests/robots/franka/test_extension.py b/tests/unit_tests/robots/franka/test_extension.py
index 43694cf70..59550a511 100644
--- a/tests/unit_tests/robots/franka/test_extension.py
+++ b/tests/unit_tests/robots/franka/test_extension.py
@@ -22,10 +22,25 @@
import gymnasium as gym
import numpy as np
+import pytest
from robots.franka.runtime_config import load_runtime_config
+def test_ray_backend_failure_is_not_a_none_subscript_error():
+ from robots.franka.env_server import _RayBackend
+
+ class Worker:
+ def get_observation(self):
+ return self
+
+ def wait(self):
+ return None
+
+ with pytest.raises(RuntimeError, match="RLinf worker failed"):
+ _RayBackend(Worker()).get_observation()
+
+
def test_franka_uses_rpent_owned_robot_config(fake_rlinf_realworld_modules):
config_path = Path(__file__).parents[4] / "robots/franka/config/example.yaml"
runtime = load_runtime_config(config_path, task_description="test task")
diff --git a/tests/unit_tests/rpent/cli/test_main_contracts.py b/tests/unit_tests/rpent/cli/test_main_contracts.py
index 8946e47d7..1bb4b35d0 100644
--- a/tests/unit_tests/rpent/cli/test_main_contracts.py
+++ b/tests/unit_tests/rpent/cli/test_main_contracts.py
@@ -111,6 +111,9 @@ def test_public_cli_help_runs_from_source_checkout_without_a_robot_runtime() ->
"--interactive",
):
assert option in result.stdout
+ assert "--dashboard-language" in result.stdout
+ assert "--dashboard-record" not in result.stdout
+ assert "--dashboard-camera-stream" not in result.stdout
@pytest.mark.parametrize(
@@ -148,6 +151,27 @@ def test_shared_cli_defaults_reach_robot_config_parser(
assert args.memory_profile == "hf"
+def test_codex_ultra_effort_reaches_robot_config_parser(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _, args = _capture_validated_args(
+ monkeypatch,
+ [
+ "--robot",
+ "libero",
+ "--planner",
+ "codex",
+ "--model",
+ "gpt-6-astra",
+ "--reasoning-effort",
+ "ultra",
+ ],
+ )
+
+ assert args.model == "gpt-6-astra"
+ assert args.reasoning_effort == "ultra"
+
+
def test_deprecated_env_alias_routes_to_the_same_robot(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -185,6 +209,21 @@ def test_robot_and_env_aliases_are_mutually_exclusive(
@pytest.mark.parametrize(
("argv", "message"),
[
+ (
+ ["--robot", "libero", "--planner", "api", "--reasoning-effort", "ultra"],
+ "requires --planner codex",
+ ),
+ (
+ [
+ "--robot",
+ "libero",
+ "--planner",
+ "claude_code",
+ "--reasoning-effort",
+ "ultra",
+ ],
+ "requires --planner codex",
+ ),
(
["--robot", "libero", "--dashboard", "--interactive"],
"cannot be used together",
@@ -356,6 +395,7 @@ def test_real_robot_terminal_requirement_fails_before_runtime(
spec = SimpleNamespace(
is_real_robot=True,
dashboard=None,
+ supports_human_interactive_exploration=False,
add_cli_args=lambda parser, use_dashboard: None,
)
monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec)
diff --git a/tests/unit_tests/rpent/cli/test_tui_contracts.py b/tests/unit_tests/rpent/cli/test_tui_contracts.py
new file mode 100644
index 000000000..decc23779
--- /dev/null
+++ b/tests/unit_tests/rpent/cli/test_tui_contracts.py
@@ -0,0 +1,110 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import queue
+import threading
+
+import pytest
+
+from rpent.cli.tui import SessionInputQueue, initial_user_message, next_user_line
+
+
+@pytest.mark.parametrize("read_line", [initial_user_message, next_user_line])
+@pytest.mark.parametrize("end_of_input", [None, "/quit", "/exit", "/q"])
+def test_scoped_input_preserves_feedback_and_real_exit(read_line, end_of_input):
+ source = queue.Queue()
+ scope = SessionInputQueue(source)
+ scope.put(" ")
+ scope.put(" inspect the other bowl ")
+ scope.put(end_of_input)
+
+ assert read_line(scope) == "inspect the other bowl"
+ assert read_line(scope) is None
+ assert source.empty()
+
+
+def test_cancelled_scope_leaves_feedback_and_real_eof_for_next_scope():
+ source = queue.Queue()
+ previous = SessionInputQueue(source)
+ previous.cancel()
+ source.put("next attempt feedback")
+ source.put(None)
+
+ assert previous.cancelled
+ assert previous.get(block=False) is None
+ assert previous.get(timeout=0) is None
+ current = SessionInputQueue(source)
+ assert not current.cancelled
+ assert current.get(block=False) == "next attempt feedback"
+ assert current.get(block=False) is None
+ assert source.empty()
+
+
+def test_nested_scope_cleanup_does_not_cancel_attended_attempt():
+ source = queue.Queue()
+ attempt = SessionInputQueue(source)
+ previous_reply = SessionInputQueue(attempt)
+ previous_reply.cancel()
+ attempt.put("continue after that reply")
+
+ assert previous_reply.get() is None
+ assert not attempt.cancelled
+ current_reply = SessionInputQueue(attempt)
+ assert next_user_line(current_reply) == "continue after that reply"
+ attempt.cancel()
+ assert current_reply.get() is None
+ assert source.empty()
+
+
+def test_scoped_input_retains_queue_nonblocking_and_timeout_contract():
+ scope = SessionInputQueue(queue.Queue())
+ with pytest.raises(queue.Empty):
+ scope.get(block=False)
+ with pytest.raises(queue.Empty):
+ scope.get(timeout=0)
+ with pytest.raises(queue.Empty):
+ scope.get(timeout=0.01)
+ with pytest.raises(ValueError, match="non-negative"):
+ scope.get(timeout=-1)
+ scope.put("available immediately")
+ assert scope.get(timeout=0) == "available immediately"
+
+
+def test_cancel_retires_waiting_reader_without_consuming_later_input():
+ reading = threading.Event()
+
+ class ObservedQueue(queue.Queue):
+ def get(self, block=True, timeout=None):
+ reading.set()
+ return super().get(block=block, timeout=timeout)
+
+ source = ObservedQueue()
+ scope = SessionInputQueue(source)
+ received = []
+ reader = threading.Thread(target=lambda: received.append(scope.get()), daemon=True)
+ reader.start()
+ try:
+ assert reading.wait(1)
+ scope.cancel()
+ source.put("new attempt")
+ reader.join(timeout=1)
+ assert not reader.is_alive()
+ assert received == [None]
+ assert source.get(block=False) == "new attempt"
+ assert source.empty()
+ finally:
+ scope.cancel()
+ reader.join(timeout=1)
diff --git a/tests/unit_tests/rpent/dashboard/test_runtime_health.py b/tests/unit_tests/rpent/dashboard/test_runtime_health.py
new file mode 100644
index 000000000..6ab22c551
--- /dev/null
+++ b/tests/unit_tests/rpent/dashboard/test_runtime_health.py
@@ -0,0 +1,45 @@
+# Copyright 2026 The RPent Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from rpent.dashboard import runtime_health
+
+
+def test_runtime_health_detects_loss_and_recovery_without_observing(monkeypatch):
+ calls = []
+ events = []
+
+ class Client:
+ available = True
+
+ def call(self, method, **kwargs):
+ calls.append(method)
+ assert kwargs["timeout_s"] == 1
+ if not self.available:
+ raise ConnectionError("env disconnected")
+ return {"status": "ok"}
+
+ class State:
+ def emit(self, event):
+ events.append(event)
+
+ client = Client()
+ monkeypatch.setattr(runtime_health, "make_rpc_client", lambda url: client)
+ monitor = runtime_health.RuntimeHealthMonitor(State(), {"env": "http://test"})
+ monitor.check()
+ client.available = False
+ monitor.check()
+ client.available = True
+ monitor.check()
+ assert [event.status for event in events] == ["ready", "failed", "ready"]
+ assert calls == ["healthz"] * 3
diff --git a/tests/unit_tests/rpent/dashboard/test_server_contracts.py b/tests/unit_tests/rpent/dashboard/test_server_contracts.py
index 340def908..e76624090 100644
--- a/tests/unit_tests/rpent/dashboard/test_server_contracts.py
+++ b/tests/unit_tests/rpent/dashboard/test_server_contracts.py
@@ -242,3 +242,85 @@ def test_healthz_still_reports_transport_liveness_only(state: DashboardState) ->
assert resp.status_code == 200
assert resp.json() == {"ok": True}
+
+
+@pytest.mark.parametrize(
+ "method,path",
+ [
+ ("GET", "/api/session/operator"),
+ ("POST", "/api/session/operator"),
+ ("POST", "/api/session/operator/finish"),
+ ],
+)
+def test_operator_control_has_no_separate_routes(state, method, path):
+ client = _client(_server(state))
+
+ assert client.request(method, path).status_code == 404
+ snapshot = client.get("/api/session/state")
+ assert snapshot.status_code == 200
+ assert snapshot.json()["operator"]["enabled"] is False
+
+
+@pytest.mark.parametrize(
+ "text",
+ [
+ "/done",
+ "/continue keep the completed half",
+ "/success",
+ "/failure missed grasp",
+ "/abort",
+ "/operator",
+ "/operator previous-request done",
+ ],
+)
+@pytest.mark.parametrize(
+ "operator_context",
+ [None, {"generation": -1, "request_id": "previous-request"}],
+)
+def test_generic_messages_do_not_enable_operator_control(state, text, operator_context):
+ state.shared_services_ready()
+ state.request_task({"seed": 1})
+ state.wait_for_task(timeout=0)
+ state.set_planner_activity("idle", accepting_input=True)
+ client = _client(_server(state))
+
+ response = client.post(
+ "/api/session/messages",
+ json={"text": text, "operator_context": operator_context},
+ )
+
+ assert response.status_code == 202
+ assert state.claim_next_pending_message().text == text
+ assert state.claim_next_pending_message() is None
+ snapshot = client.get("/api/session/state").json()
+ assert snapshot["operator"]["enabled"] is False
+ assert snapshot["operator"]["verdict"] is None
+ assert snapshot["task_generation"] == 1
+ assert not state.task_replacement_requested
+
+
+@pytest.mark.parametrize("language", ["en", "zh-cn"])
+def test_dashboard_page_uses_configured_language(state, language):
+ response = _client(_server(state, language=language)).get("/")
+
+ assert response.status_code == 200
+ assert f'' in response.text
+
+
+def test_completed_episode_video_remains_available(state):
+ state.shared_services_ready()
+ state.request_task({"seed": 1})
+ claimed = state.wait_for_task(timeout=0)
+ claimed.output_dir.mkdir(parents=True)
+ video = claimed.output_dir / "episode.mp4"
+ video.write_bytes(b"episode-video")
+ client = _client(_server(state))
+
+ assert client.get("/api/session/video").status_code == 404
+ state.complete_task(state="succeeded")
+ response = client.get("/api/session/video")
+
+ assert response.status_code == 200
+ assert response.headers["content-type"] == "video/mp4"
+ assert response.content == video.read_bytes()
+ assert client.get("/api/session/state").json()["has_video"] is True
diff --git a/tests/unit_tests/rpent/dashboard/test_session_contracts.py b/tests/unit_tests/rpent/dashboard/test_session_contracts.py
index 5aa1ef2a3..705e5dc55 100644
--- a/tests/unit_tests/rpent/dashboard/test_session_contracts.py
+++ b/tests/unit_tests/rpent/dashboard/test_session_contracts.py
@@ -14,6 +14,7 @@
from __future__ import annotations
+import argparse
from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -26,6 +27,75 @@
from rpent.robots import PromptBundle, RunConfig
+@pytest.mark.parametrize("interrupt", [False, True])
+@pytest.mark.parametrize(
+ "external_env,planner,env_endpoint",
+ [
+ (False, "codex", None),
+ (False, "codex", "http://127.0.0.1:6002"),
+ (True, "codex", "http://127.0.0.1:6002"),
+ (True, "api", "http://127.0.0.1:6002"),
+ ],
+)
+def test_attended_dashboard_startup_and_shutdown_without_recording(
+ tmp_path, monkeypatch, interrupt, external_env, planner, env_endpoint
+):
+ from rpent.cli import dashboard as dashboard_cli
+ from rpent.dashboard.runtime_health import RuntimeHealthMonitor
+ from rpent.dashboard.server import DashboardServer
+ from tests.unit_tests.rpent.dashboard.test_server_contracts import DASHBOARD_SPEC
+
+ events = []
+
+ def serve(server):
+ events.append("server")
+ assert server._language == "en"
+ return "http://127.0.0.1:8086"
+
+ def run(controller):
+ events.append("controller")
+ if interrupt:
+ raise KeyboardInterrupt
+ controller._state.request_shutdown()
+
+ monkeypatch.setattr(DashboardServer, "start", serve)
+ monkeypatch.setattr(DashboardSessionController, "run", run)
+ monkeypatch.setattr(
+ RuntimeHealthMonitor, "start", lambda self: events.append("health_start")
+ )
+ monkeypatch.setattr(
+ RuntimeHealthMonitor, "close", lambda self: events.append("health_close")
+ )
+ monkeypatch.setattr(
+ dashboard_cli, "init_output_dir", lambda path, **_kwargs: Path(path)
+ )
+ args = argparse.Namespace(
+ planner=planner,
+ env_endpoint=env_endpoint,
+ model="test-model",
+ output_dir=str(tmp_path),
+ dashboard_host="127.0.0.1",
+ dashboard_port=8086,
+ dashboard_language="en",
+ base_url=None,
+ verbose=False,
+ explore=True,
+ )
+ spec = SimpleNamespace(
+ is_real_robot=True,
+ supports_human_interactive_exploration=not external_env,
+ dashboard={**DASHBOARD_SPEC, "external_env": external_env},
+ prepare_memory=None,
+ )
+ assert (
+ dashboard_cli.run_dashboard_session(
+ args, spec, parser=argparse.ArgumentParser()
+ )
+ == 0
+ )
+ assert events == ["server", "health_start", "controller", "health_close"]
+
+
class ScriptedState:
def __init__(self, tasks: list[ClaimedTask] | None = None) -> None:
self.tasks = list(tasks or [])
@@ -175,11 +245,13 @@ def test_dashboard_session_stops_shared_daemons_in_reverse_after_cleanup_error(
@pytest.mark.parametrize("merge_fails", [False, True])
@pytest.mark.parametrize("sessions", [1, 2])
+@pytest.mark.parametrize("real_robot", [False, True])
def test_dashboard_exploration_finalizes_memory_and_reports_merge_failures(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
merge_fails: bool,
sessions: int,
+ real_robot: bool,
) -> None:
from rpent.cli import dashboard as dashboard_cli
@@ -195,6 +267,7 @@ def merge_memory(self, **kwargs: Any) -> dict[str, int]:
class FakeToolkit:
memory = FakeMemoryManager()
+ direct_verdict_requested = False
def cancel_active_and_wait(self) -> None:
pass
@@ -233,6 +306,9 @@ def unbind_toolkit(self, toolkit: Any) -> None:
def report_task_warning(self, warning: str) -> None:
self.warnings.append(warning)
+ def operator_changed(self) -> None:
+ pass
+
class FakePlanner:
def solve(self, **kwargs: Any) -> PlannerResult:
del kwargs
@@ -254,9 +330,13 @@ def solve(self, **kwargs: Any) -> PlannerResult:
finalize_run=None,
prepare_memory=None,
supports_exploration=True,
- is_real_robot=False,
+ is_real_robot=real_robot,
+ supports_human_interactive_exploration=real_robot,
parse_config=lambda args: run_config,
- init_runtime=lambda *args: ([], {}),
+ init_runtime=lambda *args: (
+ [],
+ {"options": "current-task", "task_description": "current-task"},
+ ),
prompts=PromptBundle(
system=lambda variables: "system",
user=lambda variables: "user",
@@ -284,9 +364,16 @@ def solve(self, **kwargs: Any) -> PlannerResult:
from rpent.cli import main as main_cli
monkeypatch.setattr(main_cli, "get_robot_spec", lambda name: robot_spec)
- monkeypatch.setattr(
- dashboard_cli, "get_toolkit", lambda *args, **kwargs: FakeToolkit()
- )
+
+ def get_toolkit(*args, **kwargs):
+ assert kwargs["runtime_kwargs"]["options"] == "shared-default"
+ assert kwargs["runtime_kwargs"]["task_description"] == "current-task"
+ assert kwargs["runtime_kwargs"]["shared_client"] == "client"
+ if real_robot:
+ assert kwargs["operator_input"] is state.operator
+ return FakeToolkit()
+
+ monkeypatch.setattr(dashboard_cli, "get_toolkit", get_toolkit)
monkeypatch.setattr(
dashboard_cli, "build_planner", lambda *args, **kwargs: FakePlanner()
)
@@ -296,7 +383,7 @@ def solve(self, **kwargs: Any) -> PlannerResult:
robot_spec=robot_spec,
state=state,
claimed=claimed,
- shared_runtime_kwargs={},
+ shared_runtime_kwargs={"options": "shared-default", "shared_client": "client"},
unique_components=set(),
session_root=tmp_path / "session",
)
@@ -318,3 +405,89 @@ def solve(self, **kwargs: Any) -> PlannerResult:
)
else:
assert state.warnings == []
+
+
+@pytest.mark.parametrize("environment_success", [False, True])
+def test_external_robot_dashboard_keeps_its_own_operator_and_result_contract(
+ tmp_path, monkeypatch, environment_success
+):
+ from robots.yam.robot_spec import YAM_DASHBOARD_SPEC, get_robot_spec
+ from rpent.cli import dashboard as dashboard_cli
+ from rpent.dashboard.state import DashboardState
+
+ config = RunConfig(
+ recipe_tag="yam_t0",
+ output_dir=tmp_path / "task",
+ prompt_vars={},
+ task_desc={"robot": "yam"},
+ )
+ finalized = []
+ closed = []
+ toolkit = SimpleNamespace(
+ memory=None,
+ solved=lambda: environment_success,
+ write_recipe=lambda recipe_tag: "",
+ cancel_active_and_wait=lambda: None,
+ close=lambda: closed.append(True),
+ )
+
+ def get_toolkit(*args, **kwargs):
+ assert "operator_input" not in kwargs
+ assert kwargs["mode"] == "evaluation"
+ return toolkit
+
+ def solve(**kwargs):
+ assert "DASHBOARD OPERATOR" not in kwargs["user_message"]
+ return PlannerResult(finish_result={"success": True}, messages=[], stats={})
+
+ from dataclasses import replace
+
+ spec = replace(
+ get_robot_spec(),
+ parse_config=lambda args: config,
+ init_runtime=lambda *args: ([], {}),
+ prepare_memory=lambda args, run_config: None,
+ prompts=PromptBundle(
+ system=lambda variables: "system", user=lambda variables: "user"
+ ),
+ finalize_run=finalized.append,
+ )
+ args = SimpleNamespace(
+ verbose=False,
+ robot_name="yam",
+ explore=False,
+ auto_merge_memory=False,
+ planner="api",
+ base_url=None,
+ model="offline",
+ max_tokens=128,
+ planner_timeout_s=10,
+ reasoning_effort=None,
+ claude_code_max_budget_usd=None,
+ no_images=False,
+ max_turns=2,
+ )
+ state = DashboardState(output_dir=tmp_path, dashboard_spec=YAM_DASHBOARD_SPEC)
+ monkeypatch.setattr(dashboard_cli, "get_toolkit", get_toolkit)
+ monkeypatch.setattr(
+ dashboard_cli,
+ "build_planner",
+ lambda *args, **kwargs: SimpleNamespace(solve=solve),
+ )
+
+ error = dashboard_cli._run_dashboard_task(
+ args=args,
+ robot_spec=spec,
+ state=state,
+ claimed=ClaimedTask(number=1, request={}, output_dir=config.output_dir),
+ shared_runtime_kwargs={},
+ unique_components=set(),
+ session_root=tmp_path / "session",
+ )
+
+ assert error is None
+ assert state.operator is None
+ assert closed == [True]
+ assert len(finalized) == 1
+ assert finalized[0].environment_success is environment_success
+ assert finalized[0].agent_error is None
diff --git a/tests/unit_tests/rpent/dashboard/test_state_contracts.py b/tests/unit_tests/rpent/dashboard/test_state_contracts.py
index 425b66437..027a4dd69 100644
--- a/tests/unit_tests/rpent/dashboard/test_state_contracts.py
+++ b/tests/unit_tests/rpent/dashboard/test_state_contracts.py
@@ -38,6 +38,12 @@
from rpent.session import EnvState
from rpent.tools import Toolkit, ToolResult, tool
+
+@pytest.fixture(autouse=True)
+def tool_template_output_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("rpent.utils.templates.get_output_dir", lambda: tmp_path)
+
+
DASHBOARD_SPEC: DashboardSpec = {
"task": {
"command": "/rpent-task",
@@ -81,6 +87,125 @@ def _claim_started_task(
return claimed
+@pytest.mark.parametrize(
+ "command,verdict",
+ [
+ ("/success", "success"),
+ ("/succes", "success"),
+ ("/failure", "failure"),
+ ("/abort", "abort"),
+ ],
+)
+def test_operator_verdict_closes_only_the_matching_task(tmp_path, command, verdict):
+ state = _ready_state(tmp_path)
+ claimed = _claim_started_task(state)
+ state.operator = MagicMock()
+ state.operator.snapshot.return_value = None
+ toolkit = MagicMock()
+ toolkit.request_direct_verdict.return_value = True
+ state.bind_toolkit(toolkit)
+
+ with pytest.raises(ValueError, match="expired operator task"):
+ state.submit_input(command, operator_context={"generation": -1})
+ toolkit.request_direct_verdict.assert_not_called()
+
+ state.submit_input(command, operator_context={"generation": claimed.number})
+
+ toolkit.request_direct_verdict.assert_called_once_with(verdict, "")
+ assert state.operator_snapshot()["verdict"] == verdict
+ assert state.task_replacement_requested
+ state.operator.close.assert_called_once()
+
+
+@pytest.mark.parametrize("verdict", ["success", "failure", "abort"])
+def test_operator_verdict_cannot_discard_a_queued_task(tmp_path, verdict):
+ state = _ready_state(tmp_path)
+ claimed = _claim_started_task(state)
+ state.operator = MagicMock()
+ toolkit = MagicMock()
+ toolkit.request_direct_verdict.return_value = True
+ state.bind_toolkit(toolkit)
+ state.submit_input("/rpent-task place 2")
+
+ with pytest.raises(ValueError, match="no matching active operator task"):
+ state.operator_finish(claimed.number, verdict)
+
+ toolkit.request_direct_verdict.assert_not_called()
+ state.operator.close.assert_not_called()
+ assert state.operator_snapshot()["verdict"] is None
+ state.unbind_toolkit(toolkit)
+ state.complete_task(state="cancelled")
+ replacement = state.wait_for_task(timeout=0)
+ assert replacement is not None
+ assert replacement.number == claimed.number + 1
+ assert replacement.request == {"mode": "place", "seed": 2}
+
+
+@pytest.mark.parametrize("command", ["/done", "/success", "/failure", "/abort"])
+@pytest.mark.parametrize("web_operator", [False, True])
+def test_external_robot_operator_commands_keep_terminal_routing(
+ tmp_path, command, web_operator
+):
+ from robots.yam.robot_spec import YAM_DASHBOARD_SPEC
+
+ state = DashboardState(output_dir=tmp_path, dashboard_spec=YAM_DASHBOARD_SPEC)
+ if web_operator:
+ state.operator = MagicMock()
+ with pytest.raises(ValueError, match="Use the operator terminal"):
+ state.submit_input(command, operator_context={"generation": -1})
+ if web_operator:
+ state.operator.reply.assert_not_called()
+ state.operator.close.assert_not_called()
+ else:
+ assert state.operator is None
+
+
+def test_external_robot_continue_uses_toolkit_without_web_operator(tmp_path):
+ from robots.yam.robot_spec import YAM_DASHBOARD_SPEC
+
+ state = DashboardState(
+ output_dir=tmp_path,
+ dashboard_spec={
+ **DASHBOARD_SPEC,
+ "operator_commands": YAM_DASHBOARD_SPEC["operator_commands"],
+ },
+ )
+ state.shared_services_ready()
+ _claim_started_task(state)
+ state.set_planner_activity("idle", accepting_input=True)
+ toolkit = MagicMock(spec=Toolkit)
+ toolkit.exploration_continuation.return_value = "check episode and continue"
+ state.bind_toolkit(toolkit)
+
+ assert state.submit_input("/continue", operator_context={"generation": -1}) == {
+ "command": "/continue"
+ }
+
+ toolkit.exploration_continuation.assert_called_once_with(explicit=True)
+ assert state.claim_next_pending_message().text == "check episode and continue"
+ assert state.operator is None
+
+
+@pytest.mark.parametrize("web_operator", [False, True])
+def test_task_commands_ignore_operator_reply_context(tmp_path, web_operator):
+ state = _ready_state(tmp_path)
+ _claim_started_task(state)
+ if web_operator:
+ state.operator = MagicMock()
+ state.operator.snapshot.return_value = None
+
+ state.submit_input(
+ "/rpent-task place 2",
+ operator_context={"generation": -1, "request_id": "previous-request"},
+ )
+
+ assert state.task_replacement_requested
+ assert state.snapshot()["pending_task"] == {"label": "place / seed 2"}
+ if web_operator:
+ state.operator.reply.assert_not_called()
+ state.operator.close.assert_not_called()
+
+
def test_dashboard_task_commands_validate_and_latest_unclaimed_request_wins(
tmp_path: Path,
) -> None:
diff --git a/tests/unit_tests/rpent/flywheel/test_episode.py b/tests/unit_tests/rpent/flywheel/test_episode.py
index beb995d91..6615aea20 100644
--- a/tests/unit_tests/rpent/flywheel/test_episode.py
+++ b/tests/unit_tests/rpent/flywheel/test_episode.py
@@ -100,7 +100,9 @@ def test_failed_episode_has_no_training_prefix(tmp_path):
def test_export_uses_only_success_prefix(tmp_path):
- pytest.importorskip("lerobot")
+ pytest.importorskip(
+ "lerobot.datasets.lerobot_dataset", reason="requires the flywheel extra"
+ )
success = create_episode_writer(
{"root": tmp_path, "suite": "libero_object", "task_id": 2, "seed": 1},
@@ -136,7 +138,9 @@ def test_export_uses_only_success_prefix(tmp_path):
def test_generic_rules_control_collection_validation_and_export(tmp_path):
- pytest.importorskip("lerobot")
+ pytest.importorskip(
+ "lerobot.datasets.lerobot_dataset", reason="requires the flywheel extra"
+ )
from lerobot.datasets.lerobot_dataset import LeRobotDataset
spec = {
diff --git a/tests/unit_tests/rpent/planner/test_claude_contracts.py b/tests/unit_tests/rpent/planner/test_claude_contracts.py
index df7f5d677..406b1e5a9 100644
--- a/tests/unit_tests/rpent/planner/test_claude_contracts.py
+++ b/tests/unit_tests/rpent/planner/test_claude_contracts.py
@@ -17,24 +17,32 @@
import asyncio
import json
import queue
+import threading
from pathlib import Path
from typing import Any
import claude_agent_sdk
import pytest
+from rpent.cli.tui import SessionInputQueue, next_user_line
from rpent.dashboard.events import TranscriptEvent, UsageEvent
from rpent.planner.claude_code import (
ClaudeCodePlanner,
_build_rpent_server,
_ClaudeSessionDriver,
_Recorder,
+ _TerminalSessionAdapter,
_tool_result_to_mcp,
)
from rpent.tools import ToolResult, tool
from rpent.tools.common import CommonTools
+@pytest.fixture(autouse=True)
+def tool_template_output_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("rpent.utils.templates.get_output_dir", lambda: tmp_path)
+
+
class RecordingSink:
def __init__(self) -> None:
self.events: list[Any] = []
@@ -473,3 +481,81 @@ def test_queue_and_dashboard_are_mutually_exclusive_before_sdk_use(
input_queue=queue.Queue(),
dashboard_interaction=object(),
)
+
+
+def test_terminal_adapter_close_does_not_stop_the_next_solve() -> None:
+ source = queue.Queue()
+ emitted = []
+ queries = []
+ interrupts = []
+
+ class Driver:
+ async def interrupt(self):
+ interrupts.append(True)
+
+ async def query(self, text):
+ queries.append(text)
+
+ async def run_two_solvers():
+ first = _TerminalSessionAdapter(input_queue=source, emit_user=emitted.append)
+ source.put(None)
+ await first.run(Driver())
+ await first.close()
+ assert source.empty()
+
+ second = _TerminalSessionAdapter(input_queue=source, emit_user=emitted.append)
+ source.put("inspect the next attempt")
+ source.put("/quit")
+ await second.run(Driver())
+ await second.close()
+
+ asyncio.run(run_two_solvers())
+
+ assert emitted == ["inspect the next attempt"]
+ assert queries == emitted
+ assert len(interrupts) == 3
+ assert source.empty()
+
+
+def test_terminal_adapter_cleanup_retires_delayed_reader(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ source = queue.Queue()
+ attempt = SessionInputQueue(source)
+ started = threading.Event()
+ release = threading.Event()
+ emitted = []
+ queries = []
+
+ def delayed_read(input_queue):
+ started.set()
+ assert release.wait(1)
+ return next_user_line(input_queue)
+
+ monkeypatch.setattr("rpent.planner.claude_code.next_user_line", delayed_read)
+
+ class Driver:
+ async def interrupt(self):
+ pass
+
+ async def query(self, text):
+ queries.append(text)
+
+ async def run():
+ adapter = _TerminalSessionAdapter(input_queue=attempt, emit_user=emitted.append)
+ pump = asyncio.create_task(adapter.run(Driver()))
+ try:
+ assert await asyncio.to_thread(started.wait, 1)
+ source.put("feedback for the next reply")
+ await adapter.close()
+ finally:
+ release.set()
+ await asyncio.wait_for(pump, timeout=1)
+
+ asyncio.run(run())
+
+ assert emitted == []
+ assert queries == []
+ assert not attempt.cancelled
+ assert attempt.get(block=False) == "feedback for the next reply"
+ assert source.empty()
diff --git a/tests/unit_tests/rpent/planner/test_codex_contracts.py b/tests/unit_tests/rpent/planner/test_codex_contracts.py
index 60b7171e8..1a6ca0543 100644
--- a/tests/unit_tests/rpent/planner/test_codex_contracts.py
+++ b/tests/unit_tests/rpent/planner/test_codex_contracts.py
@@ -163,6 +163,275 @@ def codex_config(**kwargs: Any) -> dict[str, Any]:
return configs
+def test_interactive_session_cleanup_does_not_poison_next_session(
+ tmp_path, monkeypatch
+):
+ import queue
+ import threading
+
+ install_fake_backend(monkeypatch)
+ inputs = queue.Queue()
+ for index in range(2):
+ delivered = threading.Event()
+
+ def steer(turn, text):
+ turn.steered.append(text)
+ delivered.set()
+
+ def stream(turn):
+ assert delivered.wait(2)
+ yield from turn.events
+
+ monkeypatch.setattr(FakeTurn, "steer", steer)
+ monkeypatch.setattr(FakeTurn, "stream", stream)
+ inputs.put(f"focus on later stages {index}")
+ planner = make_planner(tmp_path, RecordingSink(), timeout_s=5)
+ planner.solve(
+ system_prompt="test",
+ user_message="task",
+ toolkit=FakeToolkit(),
+ max_turns=10,
+ input_queue=inputs,
+ )
+ assert FakeCodex.instances[-1].thread.fake_turn.steered == [
+ f"focus on later stages {index}"
+ ]
+ assert inputs.empty()
+ assert not any(
+ t.name == "codex-steer" and t.is_alive() for t in threading.enumerate()
+ )
+
+
+def test_borrowed_mcp_server_survives_multiple_planner_replies(tmp_path, monkeypatch):
+ install_fake_backend(monkeypatch)
+ toolkit = FakeToolkit()
+ server = FakeMcpServer(toolkit)
+ planner = make_planner(tmp_path, RecordingSink())
+ for _ in range(2):
+ planner.solve(
+ system_prompt="test",
+ user_message="continue",
+ toolkit=toolkit,
+ max_turns=10,
+ mcp_server=server,
+ )
+ assert server.started and not server.stopped
+ assert len(FakeMcpServer.instances) == 1
+
+
+def test_reply_cleanup_returns_feedback_dequeued_before_cancellation(
+ tmp_path, monkeypatch
+):
+ from rpent.cli.tui import SessionInputQueue, next_user_line
+
+ install_fake_backend(monkeypatch)
+ dequeued = threading.Event()
+ cancelled = threading.Event()
+ cancel = SessionInputQueue.cancel
+
+ def read(scope):
+ line = next_user_line(scope)
+ dequeued.set()
+ assert cancelled.wait(2)
+ return line
+
+ def cancel_scope(scope):
+ cancel(scope)
+ cancelled.set()
+
+ def stream(turn):
+ assert dequeued.wait(2)
+ yield from turn.events
+
+ monkeypatch.setattr(codex_module, "next_user_line", read)
+ monkeypatch.setattr(SessionInputQueue, "cancel", cancel_scope)
+ monkeypatch.setattr(FakeTurn, "stream", stream)
+ inputs = queue.Queue()
+ inputs.put("Feedback entered as the reply ended")
+ result = make_planner(tmp_path, RecordingSink()).solve(
+ system_prompt="test",
+ user_message="task",
+ toolkit=FakeToolkit(),
+ max_turns=10,
+ input_queue=inputs,
+ )
+ assert result.error is None
+ turn = FakeCodex.instances[-1].thread.fake_turn
+ assert turn.steered == []
+ assert turn.interrupt_calls == 0
+ assert inputs.get_nowait() == "Feedback entered as the reply ended"
+ assert inputs.empty()
+
+
+@pytest.mark.parametrize("verdict", ["success", "failure", "abort"])
+@pytest.mark.parametrize("phase", ["active", "reply_returned", "operator_request"])
+def test_attended_verdict_does_not_cancel_next_codex_attempt(
+ tmp_path, monkeypatch, verdict, phase
+):
+ from rpent.cli import main as cli
+ from rpent.cli.tui import SessionInputQueue
+ from rpent.tools.human_in_the_loop import HumanInTheLoopInput
+
+ install_fake_backend(monkeypatch)
+ inputs = queue.Queue()
+ attempt_input = SessionInputQueue(inputs)
+ broker = HumanInTheLoopInput(interactive=True)
+ ready = threading.Event()
+ interrupted = threading.Event()
+ request_ready = threading.Event()
+
+ class AttendedToolkit(FakeToolkit):
+ direct_verdict_requested = False
+
+ def wait_active(self):
+ pass
+
+ toolkit = AttendedToolkit()
+
+ def accept_verdict(outcome, notes):
+ assert outcome == verdict
+ toolkit.direct_verdict_requested = True
+ attempt_input.cancel()
+ broker.cancel_pending()
+ return True
+
+ broker.bind_verdict(accept_verdict)
+ request = None
+ if phase == "operator_request":
+ monkeypatch.setattr("builtins.print", lambda *a, **kw: request_ready.set())
+ request = threading.Thread(
+ target=lambda: broker.request("Evaluate?", lambda: None, kind="verdict")
+ )
+ request.start()
+ assert request_ready.wait(2)
+
+ original_info = cli.logger.info
+
+ def info(message, *args, **kwargs):
+ if message.startswith("Waiting for"):
+ ready.set()
+ original_info(message, *args, **kwargs)
+
+ monkeypatch.setattr(cli.logger, "info", info)
+
+ def interrupt(turn):
+ turn.interrupt_calls += 1
+ interrupted.set()
+
+ def first_stream(turn):
+ if phase == "active":
+ ready.set()
+ assert interrupted.wait(2)
+ yield from turn.events
+
+ monkeypatch.setattr(FakeTurn, "interrupt", interrupt)
+ monkeypatch.setattr(FakeTurn, "stream", first_stream)
+ planner = make_planner(tmp_path, RecordingSink(), timeout_s=5)
+ results = []
+ runner = threading.Thread(
+ target=lambda: results.append(
+ cli._solve_attended_turns(
+ planner,
+ operator_input=broker,
+ state_output_dir=tmp_path / "attempt1",
+ toolkit=toolkit,
+ input_queue=attempt_input,
+ system_prompt="test",
+ user_message="task",
+ max_turns=10,
+ )
+ )
+ )
+ runner.start()
+ try:
+ assert ready.wait(2)
+ assert broker.route_line(f"/{verdict}")
+ # Feedback entered at the boundary must survive for the next attempt.
+ inputs.put("Focus on the later stages")
+ runner.join(3)
+ assert not runner.is_alive()
+ assert len(results) == 1 and results[0].error is None
+ assert FakeCodex.instances[0].thread.fake_turn.interrupt_calls == (
+ 1 if phase == "active" else 0
+ )
+ finally:
+ attempt_input.cancel()
+ toolkit.direct_verdict_requested = True
+ broker.close()
+ interrupted.set()
+ runner.join(3)
+ if request is not None:
+ request.join(2)
+ assert not request.is_alive()
+
+ delivered = threading.Event()
+
+ def steer(turn, text):
+ turn.steered.append(text)
+ delivered.set()
+
+ def next_stream(turn):
+ assert delivered.wait(2)
+ yield from turn.events
+
+ monkeypatch.setattr(FakeTurn, "steer", steer)
+ monkeypatch.setattr(FakeTurn, "stream", next_stream)
+ result = planner.solve(
+ system_prompt="test",
+ user_message="next attempt",
+ toolkit=FakeToolkit(),
+ max_turns=10,
+ input_queue=SessionInputQueue(inputs),
+ )
+ assert result.error is None
+ next_turn = FakeCodex.instances[-1].thread.fake_turn
+ assert next_turn.steered == ["Focus on the later stages"]
+ assert next_turn.interrupt_calls == 0
+ assert inputs.empty()
+
+
+@pytest.mark.parametrize("end_input", [None, "/quit", "/exit"])
+def test_terminal_eof_and_quit_interrupt_codex(tmp_path, monkeypatch, end_input):
+ install_fake_backend(monkeypatch)
+ interrupted = threading.Event()
+
+ def interrupt(turn):
+ turn.interrupt_calls += 1
+ interrupted.set()
+
+ def stream(turn):
+ assert interrupted.wait(2)
+ yield from turn.events
+
+ monkeypatch.setattr(FakeTurn, "interrupt", interrupt)
+ monkeypatch.setattr(FakeTurn, "stream", stream)
+ inputs = queue.Queue()
+ inputs.put(end_input)
+ result = make_planner(tmp_path, RecordingSink()).solve(
+ system_prompt="test",
+ user_message="task",
+ toolkit=FakeToolkit(),
+ max_turns=10,
+ input_queue=inputs,
+ )
+ assert result.error is None
+ assert FakeCodex.instances[-1].thread.fake_turn.interrupt_calls == 1
+ assert inputs.empty()
+
+
+def test_zero_timeout_disables_deadline(tmp_path, monkeypatch):
+ install_fake_backend(monkeypatch)
+ planner = make_planner(tmp_path, RecordingSink(), timeout_s=0)
+ assert planner._timeout_s is None
+ result = planner.solve(
+ system_prompt="test", user_message="task", toolkit=FakeToolkit(), max_turns=10
+ )
+ assert result.error is None
+ assert FakeCodex.instances[-1].thread.fake_turn.interrupt_calls == 0
+ with pytest.raises(ValueError, match="nonnegative"):
+ make_planner(tmp_path, RecordingSink(), timeout_s=-1)
+
+
def test_mcp_content_conversion_preserves_text_images_and_error_status() -> None:
plain = mcp_result(ToolResult(data={"value": "plain"}))
assert plain["isError"] is False
@@ -197,10 +466,14 @@ def test_config_overrides_normalize_provider_url(
) -> None:
monkeypatch.delenv("CODEX_MODEL_CONTEXT_WINDOW", raising=False)
monkeypatch.delenv("CODEX_AUTO_COMPACT_TOKEN_LIMIT", raising=False)
+ monkeypatch.delenv("CODEX_MCP_TOOL_TIMEOUT_S", raising=False)
assert _codex_mcp_config_overrides(
mcp_url="http://fake.invalid/mcp/",
base_url=None,
- ) == ['mcp_servers.rpent.url="http://fake.invalid/mcp/"']
+ ) == [
+ 'mcp_servers.rpent.url="http://fake.invalid/mcp/"',
+ "mcp_servers.rpent.tool_timeout_sec=86405.0",
+ ]
overrides = _codex_mcp_config_overrides(
mcp_url="http://fake.invalid/mcp/",
@@ -209,6 +482,7 @@ def test_config_overrides_normalize_provider_url(
assert overrides == [
'mcp_servers.rpent.url="http://fake.invalid/mcp/"',
+ "mcp_servers.rpent.tool_timeout_sec=86405.0",
f'model_provider="{PROVIDER_ID}"',
f'model_providers.{PROVIDER_ID}.name="{PROVIDER_ID}"',
f'model_providers.{PROVIDER_ID}.base_url="https://provider.invalid/root/v1"',
@@ -217,6 +491,25 @@ def test_config_overrides_normalize_provider_url(
]
+@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf", "bad"])
+def test_invalid_mcp_tool_deadline_is_rejected(monkeypatch, timeout):
+ monkeypatch.setenv("CODEX_MCP_TOOL_TIMEOUT_S", timeout)
+ with pytest.raises(ValueError):
+ _codex_mcp_config_overrides(mcp_url="http://fake.invalid/mcp/", base_url=None)
+
+
+def test_mcp_tool_deadline_override_is_scoped_to_rpent(monkeypatch):
+ monkeypatch.setenv("CODEX_MCP_TOOL_TIMEOUT_S", "2.5")
+ overrides = _codex_mcp_config_overrides(
+ mcp_url="http://fake.invalid/mcp/", base_url=None
+ )
+ assert "mcp_servers.rpent.tool_timeout_sec=7.5" in overrides
+ assert not any(
+ "tool_timeout" in item
+ for item in _codex_mcp_config_overrides(mcp_url=None, base_url=None)
+ )
+
+
def test_config_overrides_include_optional_context_limits(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -335,9 +628,11 @@ def codex_config(**kwargs: Any) -> dict[str, Any]:
assert os.environ["no_proxy"] == ".corp.invalid"
+@pytest.mark.parametrize("effort", ["none", "ultra"])
def test_planner_forwards_configured_service_tier(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
+ effort: str,
) -> None:
install_fake_backend(monkeypatch)
monkeypatch.setenv("CODEX_SERVICE_TIER", "fast")
@@ -348,7 +643,14 @@ def test_planner_forwards_configured_service_tier(
},
]
- make_planner(tmp_path, RecordingSink()).solve(
+ CodexPlanner(
+ output_dir=str(tmp_path),
+ repo_root=tmp_path,
+ model="gpt-6-astra",
+ reasoning_effort=effort,
+ timeout_s=1,
+ dashboard_events=RecordingSink(),
+ ).solve(
system_prompt="system rules",
user_message="user task",
toolkit=FakeToolkit(),
@@ -356,7 +658,9 @@ def test_planner_forwards_configured_service_tier(
)
fake_codex = FakeCodex.instances[0]
+ assert fake_codex.thread_options["model"] == "gpt-6-astra"
assert fake_codex.thread_options["service_tier"] == "fast"
+ assert fake_codex.thread.turn_prompts[0][1]["effort"].value == effort
assert fake_codex.thread.turn_prompts[0][1]["service_tier"] == "fast"
diff --git a/tests/unit_tests/rpent/planner/test_http_mcp_server.py b/tests/unit_tests/rpent/planner/test_http_mcp_server.py
index 12248ae03..29ab1248a 100644
--- a/tests/unit_tests/rpent/planner/test_http_mcp_server.py
+++ b/tests/unit_tests/rpent/planner/test_http_mcp_server.py
@@ -15,20 +15,28 @@
from __future__ import annotations
import asyncio
+import itertools
import json
+import threading
import time
+from collections.abc import Callable
from pathlib import Path
+from types import SimpleNamespace
from typing import Any
import httpx
+import pytest
from mcp import ClientSession
from mcp.client.streamable_http import streamable_http_client
+from mcp.server.lowlevel import Server
from rpent.dashboard.events import DashboardEventSink
from rpent.memory.manager import MemoryManager
+from rpent.planner.utils import http_mcp_server
from rpent.planner.utils.http_mcp_server import HttpMcpServer
from rpent.session import EnvState
from rpent.tools import Toolkit, ToolResult, tool
+from rpent.tools.human_in_the_loop import HumanInTheLoopInput
from rpent.utils.logging import init_output_dir
CONCURRENT_CALLS = [
@@ -38,6 +46,7 @@
("read_text_file", {"path": "robots/libero/guides/env_calibration.md"}),
("view_env_state", {"step": 0}),
]
+_REQUEST_IDS = itertools.count(1)
class RecordingSink(DashboardEventSink):
@@ -66,7 +75,7 @@ def __init__(self, state_dir: Path) -> None:
def _register_fake_tools(self) -> None:
@tool(readonly=True)
- def read_text_file(path: str, max_chars: int = 40000) -> dict:
+ def read_text_file(path: str, max_chars: int = 40000) -> ToolResult:
time.sleep(0.05)
p = Path(path)
return ToolResult(
@@ -74,12 +83,12 @@ def read_text_file(path: str, max_chars: int = 40000) -> dict:
)
@tool(readonly=True)
- def list_dir(path: str = "") -> dict:
+ def list_dir(path: str = "") -> ToolResult:
time.sleep(0.05)
return ToolResult(data={"path": path, "count": 0, "files": []})
@tool
- def view_env_state(step: int = -1) -> dict:
+ def view_env_state(step: int = -1) -> ToolResult:
time.sleep(0.3)
return ToolResult(data={"step": step, "mode": "evaluation"})
@@ -87,8 +96,14 @@ def view_env_state(step: int = -1) -> dict:
self.add_tool(list_dir, replace=True)
self.add_tool(view_env_state)
- def execute_tool(self, name: str, input_dict: dict[str, Any]) -> Any:
- result = super().execute_tool(name, input_dict)
+ def execute_tool(
+ self,
+ name: str,
+ input_dict: dict[str, Any],
+ *,
+ cancel_event: threading.Event | None = None,
+ ) -> ToolResult:
+ result = super().execute_tool(name, input_dict, cancel_event=cancel_event)
if result.data.get("error") == "another tool operation is still active":
self.overlap_errors.append((name, dict(input_dict)))
return result
@@ -99,13 +114,39 @@ def get_env_state(
command: dict[str, Any],
result: dict[str, Any],
elapsed_s: float,
- ) -> dict[str, Any]:
+ ) -> ToolResult:
return ToolResult(data={"observed": True})
def solved(self) -> bool:
return False
+def test_stop_timeout_retains_references_for_retry(tmp_path):
+ from types import SimpleNamespace
+
+ server = HttpMcpServer(FakeToolkit(tmp_path))
+
+ class Thread:
+ alive = True
+
+ def join(self, timeout):
+ pass
+
+ def is_alive(self):
+ return self.alive
+
+ thread = Thread()
+ uvicorn_server = SimpleNamespace(should_exit=False)
+ server._thread = thread
+ server._server = uvicorn_server
+ server.stop(timeout_s=0)
+ assert server._thread is thread and server._server is uvicorn_server
+ assert uvicorn_server.should_exit
+ thread.alive = False
+ server.stop(timeout_s=0)
+ assert server._thread is None and server._server is None
+
+
def test_http_mcp_server_serializes_concurrent_tool_calls(tmp_path: Path) -> None:
init_output_dir(tmp_path / "log")
toolkit = FakeToolkit(tmp_path)
@@ -142,3 +183,393 @@ async def _fire_concurrent(url: str) -> int:
):
rejected += 1
return rejected
+
+
+def test_http_mcp_preserves_native_validation_and_recovers_for_next_call(tmp_path):
+ init_output_dir(tmp_path / "log")
+ toolkit = FakeToolkit(tmp_path)
+ invalid = {"path": 123, "unexpected": True}
+ expected_error = toolkit.execute_tool("read_text_file", invalid)
+ assert expected_error.is_error
+ server = HttpMcpServer(toolkit)
+
+ async def run(url):
+ async with httpx.AsyncClient(trust_env=False) as http_client:
+ async with streamable_http_client(url, http_client=http_client) as (
+ read,
+ write,
+ _get_session_id,
+ ):
+ async with ClientSession(read, write) as session:
+ await session.initialize()
+ declarations = await session.list_tools()
+ published = next(
+ item
+ for item in declarations.tools
+ if item.name == "read_text_file"
+ )
+ native = next(
+ item
+ for item in toolkit.list_tools()
+ if item.name == "read_text_file"
+ )
+ assert published.inputSchema == native.input_schema
+ rejected = await session.call_tool("read_text_file", invalid)
+ assert rejected.isError
+ assert json.loads(rejected.content[0].text) == json.loads(
+ expected_error.to_text()
+ )
+ valid = await session.call_tool("read_text_file", {"path": "valid"})
+ assert not valid.isError
+ assert json.loads(valid.content[0].text)["path"] == "valid"
+
+ try:
+ asyncio.run(run(server.start()))
+ finally:
+ server.stop()
+ assert toolkit.overlap_errors == []
+
+
+class OperatorToolkit(FakeToolkit):
+ """Wait for a real input broker before counting a fake reset."""
+
+ def __init__(self, state_dir: Path) -> None:
+ super().__init__(state_dir)
+ self.operator = HumanInTheLoopInput(interactive=True)
+ self.resets = 0
+ self.probe_calls = 0
+ self.cleanup_started = threading.Event()
+ self.cleanup_release = threading.Event()
+ self.cleanup_release.set()
+
+ @tool(readonly=True)
+ def request_scene_reset() -> ToolResult:
+ """Wait for confirmation before a fake reset."""
+ try:
+ response = self.operator.request(
+ "Confirm a fake reset.", self.raise_if_cancelled, kind="reset"
+ )
+ self.raise_if_cancelled()
+ if response is not None and response.split()[0] == "done":
+ self.resets += 1
+ return ToolResult(data={"resets": self.resets})
+ finally:
+ self.cleanup_started.set()
+ assert self.cleanup_release.wait(5), (
+ "test worker cleanup was not released"
+ )
+
+ self.add_tool(request_scene_reset)
+
+ @tool(readonly=True)
+ def probe() -> ToolResult:
+ """Count an observation request."""
+ self.probe_calls += 1
+ return ToolResult(data={"observed": True})
+
+ self.add_tool(probe)
+
+
+@pytest.fixture
+def operator_server(tmp_path: Path):
+ init_output_dir(tmp_path / "log")
+ toolkit = OperatorToolkit(tmp_path)
+ server = HttpMcpServer(toolkit)
+ try:
+ yield toolkit, server.start()
+ finally:
+ toolkit.cleanup_release.set()
+ toolkit.operator.cancel_pending()
+ toolkit.cancel_active_and_wait()
+ server.stop()
+
+
+async def _wait_until(predicate: Callable[[], bool]) -> None:
+ async def poll() -> None:
+ while not predicate():
+ await asyncio.sleep(0.01)
+
+ await asyncio.wait_for(poll(), timeout=3)
+
+
+async def _call_over_http(
+ client: httpx.AsyncClient,
+ url: str,
+ name: str,
+ *,
+ timeout: float = 5,
+) -> dict[str, Any]:
+ await _initialize_http_session(client, url)
+ response = await client.post(
+ url,
+ json={
+ "jsonrpc": "2.0",
+ "id": next(_REQUEST_IDS),
+ "method": "tools/call",
+ "params": {"name": name, "arguments": {}},
+ },
+ headers={"Accept": "application/json, text/event-stream"},
+ timeout=timeout,
+ )
+ response.raise_for_status()
+ return response.json()["result"]
+
+
+async def _initialize_http_session(client: httpx.AsyncClient, url: str) -> None:
+ if "Mcp-Session-Id" in client.headers:
+ return
+ client.headers["Accept"] = "application/json, text/event-stream"
+ response = await client.post(
+ url,
+ json={
+ "jsonrpc": "2.0",
+ "id": next(_REQUEST_IDS),
+ "method": "initialize",
+ "params": {
+ "protocolVersion": "2025-03-26",
+ "capabilities": {},
+ "clientInfo": {"name": "cancellation-test", "version": "1"},
+ },
+ },
+ )
+ response.raise_for_status()
+ client.headers["Mcp-Session-Id"] = response.headers["Mcp-Session-Id"]
+ client.headers["Mcp-Protocol-Version"] = response.json()["result"][
+ "protocolVersion"
+ ]
+ response = await client.post(
+ url,
+ json={"jsonrpc": "2.0", "method": "notifications/initialized"},
+ )
+ response.raise_for_status()
+
+
+def test_http_disconnect_retires_operator_request_without_late_reset(operator_server):
+ toolkit, url = operator_server
+
+ async def run() -> None:
+ async with httpx.AsyncClient(trust_env=False) as client:
+ request = asyncio.create_task(
+ _call_over_http(client, url, "request_scene_reset", timeout=0.25)
+ )
+ await _wait_until(lambda: toolkit.operator.pending_kind == "reset")
+ request_id = toolkit.operator._pending[0]
+ with pytest.raises(httpx.ReadTimeout):
+ await request
+ await _wait_until(lambda: toolkit.operator.pending_kind is None)
+ assert toolkit.operator.route_line(f"/operator {request_id} done")
+ assert toolkit.operator.route_line("/done")
+ assert toolkit.resets == 0
+
+ next_request = asyncio.create_task(
+ _call_over_http(client, url, "request_scene_reset")
+ )
+ await _wait_until(lambda: toolkit.operator.pending_kind == "reset")
+ assert toolkit.operator._pending[0] != request_id
+ toolkit.operator.route_line(f"/operator {request_id} done")
+ assert toolkit.operator.pending_kind == "reset"
+ toolkit.operator.route_line("/done current request confirmed")
+ result = await next_request
+ assert not result.get("isError", False)
+ assert toolkit.resets == 1
+ assert toolkit.operator.pending_kind is None
+
+ asyncio.run(run())
+
+
+def test_http_disconnect_keeps_serialization_until_worker_settles(operator_server):
+ toolkit, url = operator_server
+ toolkit.cleanup_release.clear()
+
+ async def run() -> None:
+ async with httpx.AsyncClient(trust_env=False) as client:
+ request = asyncio.create_task(
+ _call_over_http(client, url, "request_scene_reset", timeout=0.25)
+ )
+ await _wait_until(lambda: toolkit.operator.pending_kind == "reset")
+ with pytest.raises(httpx.ReadTimeout):
+ await request
+ await _wait_until(toolkit.cleanup_started.is_set)
+ probe = asyncio.create_task(_call_over_http(client, url, "probe"))
+ await asyncio.sleep(0.1)
+ assert not probe.done()
+ assert toolkit.probe_calls == 0
+ toolkit.cleanup_release.set()
+ result = await probe
+ assert not result.get("isError", False)
+ assert toolkit.probe_calls == 1
+ assert toolkit.overlap_errors == []
+ assert toolkit.resets == 0
+
+ asyncio.run(run())
+
+
+def test_queued_http_disconnect_does_not_cancel_active_operator_request(
+ operator_server,
+):
+ toolkit, url = operator_server
+
+ async def run() -> None:
+ async with httpx.AsyncClient(trust_env=False) as client:
+ request = asyncio.create_task(
+ _call_over_http(client, url, "request_scene_reset")
+ )
+ await _wait_until(lambda: toolkit.operator.pending_kind == "reset")
+ with pytest.raises(httpx.ReadTimeout):
+ await _call_over_http(client, url, "probe", timeout=0.1)
+ await asyncio.sleep(0.1)
+ assert toolkit.operator.pending_kind == "reset"
+ assert toolkit.probe_calls == 0
+ toolkit.operator.route_line("/done")
+ result = await request
+ assert not result.get("isError", False)
+ await asyncio.sleep(0.1)
+ assert toolkit.probe_calls == 0
+ assert toolkit.resets == 1
+ result = await _call_over_http(client, url, "probe")
+ assert not result.get("isError", False)
+ assert toolkit.probe_calls == 1
+
+ asyncio.run(run())
+
+
+@pytest.mark.parametrize("cancel_twice", [False, True])
+def test_cancelled_mcp_coroutine_waits_for_sync_worker_cleanup(
+ tmp_path, monkeypatch, cancel_twice
+):
+ init_output_dir(tmp_path / "log")
+ toolkit = OperatorToolkit(tmp_path)
+ toolkit.cleanup_release.clear()
+ handlers = []
+
+ async def receive() -> dict[str, Any]:
+ await asyncio.Event().wait()
+ return {"type": "http.disconnect"}
+
+ class CapturingServer(Server):
+ @property
+ def request_context(self):
+ return SimpleNamespace(request=SimpleNamespace(receive=receive))
+
+ def call_tool(self, **kwargs):
+ register = super().call_tool(**kwargs)
+
+ def capture(handler):
+ handlers.append(handler)
+ return register(handler)
+
+ return capture
+
+ monkeypatch.setattr(http_mcp_server, "Server", CapturingServer)
+ http_mcp_server._build_asgi_app(toolkit)
+ call_tool = handlers[0]
+
+ async def run() -> None:
+ request = asyncio.create_task(call_tool("request_scene_reset", {}))
+ try:
+ await _wait_until(lambda: toolkit.operator.pending_kind == "reset")
+ request.cancel()
+ await _wait_until(toolkit.cleanup_started.is_set)
+ if cancel_twice:
+ request.cancel()
+ await asyncio.sleep(0.05)
+ assert toolkit.operator.pending_kind is None
+ assert not request.done()
+ probe = asyncio.create_task(call_tool("probe", {}))
+ await asyncio.sleep(0.1)
+ assert not probe.done()
+ assert toolkit.probe_calls == 0
+ toolkit.cleanup_release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await request
+ assert not (await probe).isError
+ assert toolkit.probe_calls == 1
+ assert toolkit.resets == 0
+ assert toolkit.overlap_errors == []
+ finally:
+ toolkit.cleanup_release.set()
+ toolkit.operator.cancel_pending()
+ toolkit.cancel_active_and_wait()
+ await asyncio.gather(request, return_exceptions=True)
+
+ asyncio.run(run())
+
+
+def test_mcp_cancel_notification_retires_confirmation_without_http_disconnect(
+ operator_server,
+):
+ toolkit, url = operator_server
+
+ async def run() -> None:
+ async with httpx.AsyncClient(trust_env=False) as client:
+ await _initialize_http_session(client, url)
+ request_id = next(_REQUEST_IDS)
+ request = asyncio.create_task(
+ client.post(
+ url,
+ json={
+ "jsonrpc": "2.0",
+ "id": request_id,
+ "method": "tools/call",
+ "params": {"name": "request_scene_reset", "arguments": {}},
+ },
+ )
+ )
+ await _wait_until(lambda: toolkit.operator.pending_kind == "reset")
+ operator_id = toolkit.operator._pending[0]
+ response = await client.post(
+ url,
+ json={
+ "jsonrpc": "2.0",
+ "method": "notifications/cancelled",
+ "params": {"requestId": request_id, "reason": "tool deadline"},
+ },
+ )
+ response.raise_for_status()
+ response = await request
+ response.raise_for_status()
+ assert response.json()["error"]["message"] == "Request cancelled"
+ await _wait_until(lambda: toolkit.operator.pending_kind is None)
+ toolkit.operator.route_line(f"/operator {operator_id} done")
+ toolkit.operator.route_line("/done")
+ assert toolkit.resets == 0
+ result = await _call_over_http(client, url, "probe")
+ assert not result.get("isError", False)
+ assert toolkit.probe_calls == 1
+ assert toolkit.overlap_errors == []
+
+ asyncio.run(run())
+
+
+def test_server_tool_deadline_retires_confirmation_with_client_still_connected(
+ tmp_path, monkeypatch
+):
+ monkeypatch.setenv("CODEX_MCP_TOOL_TIMEOUT_S", "0.1")
+ init_output_dir(tmp_path / "log")
+ toolkit = OperatorToolkit(tmp_path)
+ server = HttpMcpServer(toolkit)
+
+ async def run(url: str) -> None:
+ async with httpx.AsyncClient(trust_env=False) as client:
+ request = asyncio.create_task(
+ _call_over_http(client, url, "request_scene_reset")
+ )
+ await _wait_until(lambda: toolkit.operator.pending_kind == "reset")
+ operator_id = toolkit.operator._pending[0]
+ result = await request
+ assert result["isError"]
+ assert json.loads(result["content"][0]["text"])["code"] == "tool_cancelled"
+ assert toolkit.operator.pending_kind is None
+ toolkit.operator.route_line(f"/operator {operator_id} done")
+ toolkit.operator.route_line("/done")
+ assert toolkit.resets == 0
+ result = await _call_over_http(client, url, "probe")
+ assert not result.get("isError", False)
+ assert toolkit.probe_calls == 1
+
+ try:
+ asyncio.run(run(server.start()))
+ finally:
+ toolkit.operator.cancel_pending()
+ toolkit.cancel_active_and_wait()
+ server.stop()
diff --git a/tests/unit_tests/rpent/robots/components/test_pi05_vla_server_contracts.py b/tests/unit_tests/rpent/robots/components/test_pi05_vla_server_contracts.py
index 4c5bdcbd2..5b2e56458 100644
--- a/tests/unit_tests/rpent/robots/components/test_pi05_vla_server_contracts.py
+++ b/tests/unit_tests/rpent/robots/components/test_pi05_vla_server_contracts.py
@@ -42,6 +42,9 @@ def cuda(self):
def eval(self):
return self
+ def parameters(self):
+ return iter([torch.zeros(1)])
+
def predict_action_batch(self, obs, mode):
calls.append(mode)
shape = (20, 20) if embodiment == "dual_franka" else (1, 30, 14)
@@ -85,30 +88,29 @@ def get_model(cfg, torch_dtype):
@pytest.mark.parametrize("embodiment", ["libero", "dual_franka", "robodojo", "yam"])
-def test_cli_forwards_model_configuration(monkeypatch, embodiment):
+@pytest.mark.parametrize("repo_id", [None, "test/dataset"])
+def test_cli_forwards_model_configuration(monkeypatch, embodiment, repo_id):
import sys
from types import SimpleNamespace
from rpent.robots.components import pi05_vla_server as server
received = {}
- monkeypatch.setattr(
- sys,
- "argv",
- [
- "pi05_vla_server",
- "--embodiment",
- embodiment,
- "--model-path",
- "/checkpoint",
- "--repo-id",
- "test/dataset",
- "--norm-stats-path",
- "/stats",
- "--port",
- "6000",
- ],
- )
+ monkeypatch.setenv("DUAL_FRANKA_REPO_ID", "unrelated/local-deployment")
+ argv = [
+ "pi05_vla_server",
+ "--embodiment",
+ embodiment,
+ "--model-path",
+ "/checkpoint",
+ "--norm-stats-path",
+ "/stats",
+ "--port",
+ "6000",
+ ]
+ if repo_id is not None:
+ argv.extend(["--repo-id", repo_id])
+ monkeypatch.setattr(sys, "argv", argv)
def facade(**kwargs):
received.update(kwargs)
@@ -117,7 +119,7 @@ def facade(**kwargs):
monkeypatch.setattr(server, "Pi05VLAFacade", facade)
server.main()
assert received["embodiment"] == embodiment
- assert received["repo_id"] == "test/dataset"
+ assert received["repo_id"] == repo_id
assert received["norm_stats_path"] == "/stats"
assert received["port"] == 6000
diff --git a/tests/unit_tests/rpent/robots/test_config_contracts.py b/tests/unit_tests/rpent/robots/test_config_contracts.py
index f98c8acc0..803b027a7 100644
--- a/tests/unit_tests/rpent/robots/test_config_contracts.py
+++ b/tests/unit_tests/rpent/robots/test_config_contracts.py
@@ -376,8 +376,17 @@ def test_robot_config_flag_is_recorded_at_parse_time(
# The packaged defaults reference on-robot easy_handeye YAMLs: parse
# fails fast, but only after the default config path is recorded.
args = _parser(robot_name).parse_args([])
- with pytest.raises(ValueError, match="easy_handeye"):
- get_robot_spec(robot_name).parse_config(args)
+ # Local deployments can have real absolute paths. Simulate missing
+ # calibration without depending on whether this host is calibrated.
+ exists = Path.exists
+ with monkeypatch.context() as missing_calibration:
+ missing_calibration.setattr(
+ Path,
+ "exists",
+ lambda path: False if path.suffix == ".yaml" else exists(path),
+ )
+ with pytest.raises(ValueError, match="easy_handeye"):
+ get_robot_spec(robot_name).parse_config(args)
assert get_robot_config_path() == packaged_default
finally:
set_robot_config_path(None)
diff --git a/tests/unit_tests/rpent/tools/test_human_in_the_loop.py b/tests/unit_tests/rpent/tools/test_human_in_the_loop.py
index 28e5eb2f2..221b6258e 100644
--- a/tests/unit_tests/rpent/tools/test_human_in_the_loop.py
+++ b/tests/unit_tests/rpent/tools/test_human_in_the_loop.py
@@ -14,14 +14,77 @@
"""Operator replies must not be consumed as planner steering, or vice versa."""
+import json
import threading
import pytest
-from rpent.tools.human_in_the_loop import HumanInTheLoopInput
+from rpent.tools.human_in_the_loop import (
+ HumanInTheLoopInput,
+ OperatorCommand,
+ parse_operator_command,
+ record_operator_outcome,
+)
from rpent.tools.toolkit import ToolCancelled
+@pytest.mark.parametrize(
+ "line,expected",
+ [
+ ("please continue", None),
+ ("success", None),
+ ("/rpent-task --task-id 5", None),
+ (" /DONE scene restored ", OperatorCommand("done", "scene restored")),
+ ("/continue keep the bowl", OperatorCommand("continue", "keep the bowl")),
+ ("/succes", OperatorCommand("success")),
+ ("/failure missed grasp", OperatorCommand("failure", "missed grasp")),
+ ("/abort", OperatorCommand("abort")),
+ (
+ "/operator request-1 continue keep the bowl",
+ OperatorCommand("operator", "keep the bowl", "request-1", "continue"),
+ ),
+ ],
+)
+def test_shared_operator_command_parser(line, expected):
+ assert parse_operator_command(line) == expected
+
+
+@pytest.mark.parametrize("line", ["/operator", "/operator request-1"])
+def test_incomplete_operator_command_never_becomes_chat(line):
+ with pytest.raises(ValueError, match="Usage: /operator"):
+ parse_operator_command(line)
+ broker = HumanInTheLoopInput(interactive=True)
+ assert broker.route_line(line)
+ assert broker.feedback_prompt() == ""
+
+
+@pytest.mark.parametrize("outcome", ["success", "failure"])
+def test_operator_outcome_preserves_evidence_and_refuses_overwrite(tmp_path, outcome):
+ verdict = {"operator_verdict": outcome, "notes": "operator evidence"}
+ messages = [{"role": "user", "content": "keep the completed part"}]
+ kwargs = {
+ "memory_root": tmp_path,
+ "recipe_tag": "dual_franka_t5",
+ "run_name": "run-1",
+ "session_number": 2,
+ "state_output_dir": tmp_path / "session_002",
+ "verdict": verdict,
+ "messages": messages,
+ "planner_error": "interrupted by verdict",
+ }
+ path = record_operator_outcome(**kwargs)
+ assert path.name == f"run-1-session-002-{outcome}.json"
+ record = json.loads(path.read_text())
+ assert record["verdict"] == verdict
+ assert record["messages"] == messages
+ assert record["session"] == 2
+ assert record["evidence_dir"] == str(tmp_path / "session_002")
+ assert record["planner_error"] == "interrupted by verdict"
+ with pytest.raises(FileExistsError):
+ record_operator_outcome(**kwargs)
+ assert json.loads(path.read_text()) == record
+
+
def test_operator_replies_are_request_scoped_and_do_not_consume_steering(monkeypatch):
broker = HumanInTheLoopInput(interactive=True)
ready = threading.Event()
@@ -43,6 +106,31 @@ def test_operator_replies_are_request_scoped_and_do_not_consume_steering(monkeyp
assert broker._pending is None
+@pytest.mark.parametrize("kind,command", [("reset", "/done"), ("verdict", "/continue")])
+def test_feedback_notes_are_returned_and_preserved(
+ tmp_path, monkeypatch, kind, command
+):
+ broker = HumanInTheLoopInput(
+ interactive=True, feedback_path=tmp_path / "feedback.jsonl"
+ )
+ ready = threading.Event()
+ result = []
+ monkeypatch.setattr("builtins.print", lambda *args, **kwargs: ready.set())
+ worker = threading.Thread(
+ target=lambda: result.append(broker.request("confirm", lambda: None, kind=kind))
+ )
+ worker.start()
+ assert ready.wait(2)
+ assert not broker.route_line("保留前半段成果，重点练后半段")
+ assert not result # Feedback alone never authorizes robot reset.
+ assert broker.route_line(command + " 篮筐中的碗盘保持不动")
+ worker.join(2)
+ assert result == [command[1:] + " 篮筐中的碗盘保持不动"]
+ assert "重点练后半段" in broker.feedback_prompt()
+ assert "篮筐中的碗盘保持不动" in broker.feedback_prompt()
+ assert len((tmp_path / "feedback.jsonl").read_text().splitlines()) == 2
+
+
def test_operator_eof_and_cancellation_release_pending_requests(monkeypatch):
broker = HumanInTheLoopInput(interactive=True)
ready = threading.Event()
@@ -123,7 +211,7 @@ def route(line):
def test_success_is_control_and_never_steering():
broker = HumanInTheLoopInput(interactive=True)
calls = []
- broker.bind_verdict(lambda verdict: calls.append(verdict) or True)
+ broker.bind_verdict(lambda verdict, notes: calls.append(verdict) or True)
assert not broker.route_line(" success ")
assert broker.route_line("/success")
assert len(calls) == 1
@@ -133,16 +221,39 @@ def test_success_is_control_and_never_steering():
assert len(calls) == 1
+@pytest.mark.parametrize("verdict", ["success", "failure", "abort"])
+def test_verdict_notes_are_control_not_steering(tmp_path, verdict):
+ broker = HumanInTheLoopInput(
+ interactive=True, feedback_path=tmp_path / "feedback.jsonl"
+ )
+ calls = []
+ broker.bind_verdict(lambda value, notes: calls.append((value, notes)) or True)
+ assert broker.route_line(f"/{verdict} 建议采用混合控制")
+ assert calls == [(verdict, "建议采用混合控制")]
+ assert "建议采用混合控制" in broker.feedback_prompt()
+
+
+def test_failed_prompt_display_does_not_leave_pending_request(monkeypatch):
+ broker = HumanInTheLoopInput(interactive=True)
+
+ def fail(*args, **kwargs):
+ raise OSError("terminal closed")
+
+ monkeypatch.setattr("builtins.print", fail)
+ with pytest.raises(OSError, match="terminal closed"):
+ broker.request("reset", lambda: None, kind="reset")
+ assert broker.pending_kind is None
+
+
def test_only_slash_verdicts_are_control_commands():
broker = HumanInTheLoopInput(interactive=True)
verdicts = []
- broker.bind_verdict(lambda verdict: verdicts.append(verdict) or True)
+ broker.bind_verdict(lambda verdict, notes: verdicts.append(verdict) or True)
for text in (
"success",
"failure",
"抓取成功了",
"success 请继续",
- "/success later",
):
assert not broker.route_line(text)
assert broker.route_line("/success")
@@ -164,7 +275,7 @@ def test_shortcuts_only_answer_matching_active_request(
ready = threading.Event()
values = []
monkeypatch.setattr("builtins.print", lambda *a, **kw: ready.set())
- assert broker.route_line(command) # no buffering before a request
+ assert broker.route_line(command) is (command != "/continue")
ready.clear()
thread = threading.Thread(
target=lambda: values.append(broker.request("test", lambda: None, kind=kind))
@@ -176,14 +287,14 @@ def test_shortcuts_only_answer_matching_active_request(
assert broker.route_line(command)
thread.join(2)
assert not thread.is_alive() and values == [answer]
- assert broker.route_line(command) # duplicate cannot authorize another operation
+ assert broker.route_line(command) is (command != "/continue")
assert not broker.route_line(answer)
def test_abort_is_control_but_bare_words_are_chat():
broker = HumanInTheLoopInput(interactive=True)
calls = []
- broker.bind_verdict(lambda verdict: calls.append(verdict) or True)
+ broker.bind_verdict(lambda verdict, notes: calls.append(verdict) or True)
assert broker.route_line("/abort")
assert calls == ["abort"]
for word in ("done", "continue", "abort", "success", "failure"):
diff --git a/tests/unit_tests/rpent/tools/test_toolkit_contracts.py b/tests/unit_tests/rpent/tools/test_toolkit_contracts.py
index c857c8a74..bdb00fac7 100644
--- a/tests/unit_tests/rpent/tools/test_toolkit_contracts.py
+++ b/tests/unit_tests/rpent/tools/test_toolkit_contracts.py
@@ -525,6 +525,71 @@ def fail() -> ToolResult:
).data.get("_finish", False)
+@pytest.mark.timeout(5)
+def test_wait_active_preserves_authorized_tool_execution(tmp_path):
+ toolkit = _ContractToolkit(tmp_path)
+ started = threading.Event()
+ release = threading.Event()
+ waiting = threading.Event()
+ settled = threading.Event()
+ results = []
+
+ @tool(readonly=True)
+ def blocking() -> ToolResult:
+ started.set()
+ assert release.wait(2)
+ toolkit.raise_if_cancelled()
+ return ToolResult(data={"completed": True})
+
+ toolkit.add_tool(blocking)
+
+ def wait():
+ waiting.set()
+ toolkit.wait_active()
+ settled.set()
+
+ worker = threading.Thread(
+ target=lambda: results.append(toolkit.execute_tool("blocking", {}))
+ )
+ waiter = threading.Thread(target=wait)
+ worker.start()
+ try:
+ assert started.wait(2)
+ waiter.start()
+ assert waiting.wait(2)
+ assert not settled.wait(0.05)
+ finally:
+ release.set()
+ worker.join(2)
+ if waiter.ident is not None:
+ waiter.join(2)
+ assert not worker.is_alive() and not waiter.is_alive()
+ assert settled.is_set()
+ assert results[0].data == {"completed": True}
+ toolkit.wait_active()
+
+
+@pytest.mark.timeout(5)
+def test_transport_cancellation_before_dispatch_never_enters_handler(tmp_path):
+ toolkit = _ContractToolkit(tmp_path)
+ calls = []
+
+ @tool(readonly=True)
+ def act() -> ToolResult:
+ calls.append("acted")
+ return ToolResult(data={"ok": True})
+
+ toolkit.add_tool(act)
+ cancelled = threading.Event()
+ cancelled.set()
+ result = toolkit.execute_tool("act", {}, cancel_event=cancelled)
+
+ assert result.data["code"] == "tool_cancelled"
+ assert calls == []
+ assert toolkit.execute_tool("act", {}).data == {"ok": True}
+ assert calls == ["acted"]
+
+
@pytest.mark.timeout(5)
def test_toolkit_cooperatively_cancels_and_cleans_up_active_operation(
tmp_path: Path,
diff --git a/tests/unit_tests/rpent/utils/test_daemon_lifecycle.py b/tests/unit_tests/rpent/utils/test_daemon_lifecycle.py
index cbd83bb6a..77ee8fbc8 100644
--- a/tests/unit_tests/rpent/utils/test_daemon_lifecycle.py
+++ b/tests/unit_tests/rpent/utils/test_daemon_lifecycle.py
@@ -219,6 +219,24 @@ def test_daemon_prepends_checkout_already_on_pythonpath(
)
+@pytest.mark.parametrize("pythonpath", [None, "/other"])
+def test_daemon_does_not_inject_an_implicit_rlinf_checkout(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ pythonpath: str | None,
+) -> None:
+ monkeypatch.delenv("RPENT_RLINF_ROOT", raising=False)
+ monkeypatch.delenv("RLINF_REPO_PATH", raising=False)
+ monkeypatch.setenv("RPENT_REPO_ROOT", str(tmp_path / "RPent"))
+ (tmp_path / "rlinf").mkdir()
+ if pythonpath is None:
+ monkeypatch.delenv("PYTHONPATH", raising=False)
+ else:
+ monkeypatch.setenv("PYTHONPATH", pythonpath)
+ daemon = ProcessDaemon("env-probe", [sys.executable, "-c", "pass"])
+ assert daemon.subprocess_env.get("PYTHONPATH") == pythonpath
+
+
def test_daemon_sets_pythonpath_when_parent_has_none(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,