fix(dual-franka): align deployment, interaction and task prompts

.github/workflows/unittest.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Install test dependencies
         run: uv pip install --system -e ".[test]"
       - name: Install Flywheel export dependencies
-        run: uv pip install --system -e ".[flywheel]" --torch-backend cpu
+        run: uv pip install --system -e ".[flywheel]" "opencv-python-headless==4.10.0.84" --torch-backend cpu
       - name: Run unit tests with coverage
         run: coverage run --source=rpent,robots -m pytest tests/unit_tests -v
       - name: Generate coverage report

robots/dual_franka/config/example.yaml

@@ -1,6 +1,7 @@
 # RPent example config for a dual-Franka setup.
 robot:
   realtime_config: ignore
+  # Omit compliance to use RLinf defaults; tune overrides in your deployment YAML.
   arms:
     left:
       ip: "172.16.0.2"
@@ -14,6 +15,8 @@ robot:
         connection: "/dev/serial/by-id/usb-FTDI_USB_TO_RS-485_DAAIT8PU-if00-port0"
 
 cameras:
+  # Wrist cameras connect to their arm's controller node; the base and
+  # perception cameras connect to the environment node.
   main: left_wrist_0_rgb
   # Agent-facing view policy.  ``inline`` views are sent directly in planner
   # tool results; ``auxiliary`` views are saved as artifacts and can be inspected
@@ -52,7 +55,8 @@ workspace:
     - [0.8, 0.8, 0.7, 3.1416, 3.1416, 3.1416]
 
 joint_health:
-  # Tune these guard rails for the mounting, reset posture, tool, and task.
+  # Advisory heuristics, not hardware limits or an automatic reset trigger.
+  # Tune for the mounting, reset posture, tool, and task.
   thresholds:
     left:
       warning_drift_l2: 2.4

robots/dual_franka/dual_franka_manual_call.py

@@ -67,7 +67,11 @@
 
 def _tool_spec_map(*, include_manual: bool = False) -> dict[str, dict[str, Any]]:
     specs = {
-        tool.name: {"name": tool.name, "input_schema": tool.input_schema}
+        tool.name: {
+            "name": tool.name,
+            "description": tool.description,
+            "input_schema": tool.input_schema,
+        }
         for tool in DualFrankaToolkit.declared_tools()
     }
     if include_manual:

robots/dual_franka/perception.py

@@ -831,12 +831,24 @@ def _mask_to_camera_world(
         result.update({"point_xyz": None, "world_error": "empty mask"})
         return result
 
+    # Keep the rejected mask visible too; a missing valid 3-D point must not
+    # hide where segmentation actually landed in the image.
+    result["centroid_pixel"] = [int(np.median(rows)), int(np.median(cols))]
+    result["mask_bbox_rc"] = [
+        int(rows.min()),
+        int(cols.min()),
+        int(rows.max()),
+        int(cols.max()),
+    ]
+
     depths = depth[rows, cols].astype(np.float64)
     valid_depth = np.isfinite(depths) & (depths > 0.0)
     rows = rows[valid_depth]
     cols = cols[valid_depth]
     depths = depths[valid_depth]
     result["valid_depth_pixels"] = int(depths.size)
+    if depths.size:
+        result["raw_depth_median_m"] = round(float(np.median(depths)), 5)
     if depths.size < min_valid:
         result.update(
             {

robots/dual_franka/prompt_bundle.py

@@ -33,7 +33,6 @@ def system_prompt(variables: Mapping[str, object] | None = None) -> PromptNode:
         "RUNTIME": system_parts.RUNTIME,
         "SAFETY RULES": Numbered(system_parts.RULES),
         "CAMERA AND PROJECTION RULES": Numbered(system_parts.CAMERA_AND_PROJECTION),
-        "VLA SEGMENT GATES": Numbered(system_parts.VLA_GATES),
         "WORKFLOW": Numbered(system_parts.WORKFLOW),
     }
     return node

robots/dual_franka/prompts/explore.py

@@ -22,31 +22,41 @@
 from rpent.prompt.utils import Numbered
 
 MODE = """MULTI-ATTEMPT REAL-ROBOT EXPLORATION. You are session {{session_number}}
-of {{session_max}} on {{recipe_tag}}. Complete the task through bounded physical
+of {{session_max}} on {{recipe_tag}}. Complete the task through physical
 actions, learn from failed attempts, and leave evidence-backed memory.
 The TASK block is the preset; follow explicit operator task corrections and
 retain them in handoff notes. Task solvability is not guaranteed. Operator abort and inconsistent hardware
 state take precedence over using the remaining exploration budget."""
 
 RULES = (
+    "Acknowledge explicit operator feedback and explain the resulting plan change. "
+    "An operator may request practicing a later task stage while preserving earlier "
+    "progress. Request confirmation of that partial starting scene, not full restoration. "
+    "Do not repeat completed stages merely to satisfy the preset order. Still obtain "
+    "fresh reset confirmation before motion, inspect the scene and re-localize. "
+    "Record partial-start practice as such, not as an end-to-end successful run. "
+    "If feedback is ambiguous, ask a specific question instead of ignoring it.",
     "Before the first motion in every session, request_scene_reset(reason, "
     "expected_scene_state). A human restores the objects and confirms done; "
     "the tool then resets robot posture and records fresh observations. A new "
     "toolkit or planner session does not restore the tabletop.",
     "After reset, re-read the new state and re-localize. Never reuse a pixel, "
     "point or TCP target from an earlier attempt. Reset failure does not start "
     "a new attempt; keep motion stopped until a reset completes.",
-    "Read describe_dual_franka_setup for registered tools and cameras. Use move_delta, "
-    "rotate_delta, gripper tools, recover_joint_posture, back_project, optional "
-    "segment, and the named vla_right_grasp/vla_handoff/vla_left_place skills. "
-    "Do not assume LIBERO primitives or the older vla_grasp tool exist.",
+    "Read describe_dual_franka_setup for registered tools and cameras. Use only "
+    "the tools available for the current task; do not assume a fixed skill set.",
     "Ask request_operator_verdict after apparent success or failure. The human "
     "answers success, failure, continue or abort. Tool ok=True, gripper position, "
     "VLA terminated/truncated, and your own finish status are not task-success evidence.",
     "Any subsequent physical action invalidates the verdict. continue clears "
-    "the previous verdict. Obtain another judgment before finish.",
+    "the previous verdict. A returned continue is already authorization to "
+    "resume this attempt, not a request to wait for another continue. Read its "
+    "notes and fresh observation and continue without resetting. Do not ask "
+    "again about the same answered question unless new evidence introduces a "
+    "concrete blocker. Obtain another judgment before finish.",
     "For a failed attempt, archive evidence and update wip before requesting "
-    "a new scene reset. Change a named lever: order, staging, target or VLA prompt/chunk budget. "
+    "a new scene reset. Change one meaningful lever, such as perception inputs "
+    "or use of available tools, within the current task requirements. "
     "Use in-place recovery only when safe. Never force a restart after operator abort.",
 )
 
@@ -119,7 +129,6 @@ def system_prompt():
         "RUNTIME": base.RUNTIME,
         "SAFETY RULES": Numbered(base.RULES),
         "CAMERA AND PROJECTION RULES": Numbered(base.CAMERA_AND_PROJECTION),
-        "VLA SEGMENT GATES": Numbered(base.VLA_GATES),
         "EXPLORATION WORKFLOW": Numbered(RULES),
         "LAYERED MEMORY": MEMORY,
     }

robots/dual_franka/prompts/system.py

@@ -41,15 +41,20 @@
     "specific historical step.",
     "Use purposeful corrections and use the configured inline view(s) as "
     "the primary visual evidence.",
+    "For move_delta and rotate_delta, status='completed' means execution "
+    "finished, not exact target attainment. target_reached and residual errors "
+    "are accuracy diagnostics; target_reached=false alone is not a controller "
+    "fault or a requirement to retry, stop, or request operator feedback. "
+    "Use the actual final pose and returned observation for the next action.",
     "Never use a VLA trained for another embodiment or action normalization.",
     "VLA segment tools accept a planner-facing prompt. A deployment may still "
     "override it with a checkpoint-specific training instruction during policy "
     "inference; inspect the tool result to see requested/effective prompts.",
     "Treat a VLA segment stop as the end of one manipulation segment, not as "
     "proof that its physical goal succeeded.",
-    "Inspect joint_health after every mutation and recover joint posture before "
-    "continuing when either arm reports warning or critical.",
-    "If state, cameras, or motion results are inconsistent, stop instead of guessing.",
+    "Stop for actual controller faults, protective motion aborts, or conflicting "
+    "state/camera evidence that prevents safe action. An ordinary residual "
+    "tracking error alone is not such a conflict.",
 )
 
 CAMERA_AND_PROJECTION = (
@@ -60,8 +65,9 @@
     "Choose a pixel well inside visible material of the named target object or "
     "target container, away from silhouettes, rims, walls, wires, occluders, and "
     "background. Never project image-space air above an object.",
-    "When SAM3 is available, use segment on a registered localization camera to "
-    "segment visible target objects or destination regions before manual pixel projection. "
+    "Choose perception tools according to the current task. When using SAM3, "
+    "call segment on a registered localization camera for the intended object "
+    "or destination region. "
     "Inspect the returned mask overlay yourself; trust the returned right_base "
     "point only when the mask and median marker cover the intended target.",
     "SAM3 text prompts are phrase-sensitive. Prefer short color/object/relation "
@@ -75,49 +81,11 @@
     "After a verified projection, trust the returned right_base xyz and TCP-to-"
     "point deltas as the primary metric evidence; do not override them with "
     "unaided 2D RGB distance guesses.",
-    "Do not move an arm or held object merely to improve camera visibility. If "
-    "visibility is insufficient, mark the state uncertain or stop for operator "
-    "feedback instead of performing active-vision motions.",
-)
-
-VLA_GATES = (
-    "Use VLA segment tools for contact-rich motion: vla_right_grasp, "
-    "vla_handoff, and vla_left_place.",
-    "vla_right_grasp ends after right gripper closure plus lift; vla_handoff "
-    "ends after right gripper opening plus a configured release delay; "
-    "vla_left_place ends after left gripper opening plus lift. These boundaries "
-    "only mean a segment ended; verify physical success from images, gripper "
-    "widths/open flags, and projection geometry before continuing.",
-    "After a grasp VLA, do not decide lifted/held status from 2D appearance "
-    "alone. Check right gripper state and the latest inline image. If held status "
-    "is unclear, project the best visible held-object surface when possible and "
-    "compare right_base z primarily with the source/table height or pre-grasp "
-    "source projection.",
-    "If a verified projection of the intended object or held-object surface is "
-    "higher than source/table height, treat the grasp as successful and proceed "
-    "to vla_handoff. Mark failure only when verified projection shows the same "
-    "object still at table height and gripper/image evidence shows the right "
-    "gripper is empty or not supporting it.",
-    "After a confirmed right-hand grasp, call vla_handoff directly. Do not use "
-    "move_delta, rotate_delta, open_gripper, close_gripper, or any other "
-    "rule-based primitive to prepare or reposition either arm for handoff; the VLA owns the bimanual "
-    "approach, relative pose adjustment, left-hand grasp, right-hand release, "
-    "and collision-aware transfer.",
-    "Before vla_left_place, first verify that vla_handoff has ended and the "
-    "left gripper holds the intended object. If the left hand is not clearly "
-    "holding the object, do not stage for placement.",
-    "For placement staging, project a clearly valid target pixel inside the "
-    "required destination, then use projected x/y only. Keep the current left "
-    "TCP z with delta_z=0 by default; do not move to the projected z coordinate "
-    "or add large vertical clearance. Let vla_left_place handle descent, "
-    "release, and post-release lift.",
 )
 
 WORKFLOW = (
     "Call describe_dual_franka_setup, then inspect the initial synchronized state.",
-    "Build a conservative localization table for the current category from the "
-    "latest inline image and keep it updated after each primitive.",
     "After each motion, inspect the returned result before choosing the next action.",
-    "Use the task's VLA segment tools for contact-rich motion.",
+    "Follow the task's policy for choosing VLA skills or direct control.",
     "Finish only when the success evidence is visible and consistent with state.",
 )

robots/dual_franka/prompts/user.py

@@ -23,6 +23,6 @@
 
 BEGIN = """Call describe_dual_franka_setup before acting. Then call
 view_env_state with step 0, inspect the configured inline visual evidence
-together with both arms' TCP/gripper/joint-health state, and execute the task
+together with both arms' TCP/gripper state, and execute the task
 conservatively with the exposed bounded tools. Auxiliary camera views are
 artifact views for targeted follow-up inspection only."""

robots/dual_franka/robot_spec.py

@@ -56,7 +56,7 @@
                 "name": "task_id",
                 "kind": "integer",
                 "minimum": 0,
-                "suggestions": tuple(sorted(DUAL_FRANKA_TASKS)),
+                "suggestions": tuple(str(key) for key in sorted(DUAL_FRANKA_TASKS)),
             },
         ),
         "display": "Dual Franka task {task_id}",
@@ -424,7 +424,10 @@ def _init_runtime(
     connectors = {
         "env": lambda rpc: {
             "env": DualFrankaEnvClient(
-                rpc, reset_on_connect=not getattr(args, "explore", False)
+                rpc,
+                reset_on_connect=not (
+                    getattr(args, "explore", False) or getattr(args, "dashboard", False)
+                ),
             ),
             "task_description": get_dual_franka_task(args.task_id).instruction,
             "vla_instruction": get_dual_franka_task(args.task_id).vla_instruction,