From 095c998c7a12df4deac6adc3dc5e2d0195a65c79 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 21:13:42 +0800 Subject: [PATCH 01/18] fix(dual-franka): fresh-observations review slice --- robots/dual_franka/perception.py | 12 +++ robots/dual_franka/toolkit.py | 27 ++++++- robots/dual_franka/tools.py | 2 +- .../dual_franka/test_dual_franka_tools.py | 73 +++++-------------- .../robots/dual_franka/test_exploration.py | 40 ++++++++++ .../robots/fixtures/tool_schemas.json | 2 +- 6 files changed, 96 insertions(+), 60 deletions(-) diff --git a/robots/dual_franka/perception.py b/robots/dual_franka/perception.py index 487185f8b..82eb54f18 100644 --- a/robots/dual_franka/perception.py +++ b/robots/dual_franka/perception.py @@ -843,12 +843,24 @@ def _mask_to_camera_world( result.update({"point_xyz": None, "world_error": "empty mask"}) return result + # Keep the rejected mask visible too; a missing valid 3-D point must not + # hide where segmentation actually landed in the image. + result["centroid_pixel"] = [int(np.median(rows)), int(np.median(cols))] + result["mask_bbox_rc"] = [ + int(rows.min()), + int(cols.min()), + int(rows.max()), + int(cols.max()), + ] + depths = depth[rows, cols].astype(np.float64) valid_depth = np.isfinite(depths) & (depths > 0.0) rows = rows[valid_depth] cols = cols[valid_depth] depths = depths[valid_depth] result["valid_depth_pixels"] = int(depths.size) + if depths.size: + result["raw_depth_median_m"] = round(float(np.median(depths)), 5) if depths.size < min_valid: result.update( { diff --git a/robots/dual_franka/toolkit.py b/robots/dual_franka/toolkit.py index 24b3a5ba0..c69a06c96 100644 --- a/robots/dual_franka/toolkit.py +++ b/robots/dual_franka/toolkit.py @@ -224,15 +224,33 @@ def _guard_motion(self, inner, **kwargs) -> ToolResult: self._clear_verdict() return inner(**kwargs) + def _view_env_state(self, step: int = -1) -> ToolResult: + if step >= 0: + return dual_franka_tools.view_env_state(step, state=self._state) + return self._observe_current("observe_current") + + def _observe_current(self, action: str) -> ToolResult: + output = self.get_env_state( + command={"action": action}, result={}, elapsed_s=0.0 + ) + self._validate_observation() + self._publish_step(self.state.latest_record()) + return output + def _current_perception(self, inner, **kwargs) -> ToolResult: step = kwargs.get("step") - if not self._scene_ready or ( - step is not None and step != -1 and step < self._attempt_start_step - ): + if not self._scene_ready: + return ToolResult( + data={ + "error": "localization refused; use fresh observations after confirmed scene reset" + } + ) + if step is not None and step != -1 and step < self.state.latest_step: return ToolResult( data={ "error": ( - "localization refused; use fresh observations after confirmed scene reset" + f"localization refused: step {step} is stale; use current step " + f"{self.state.latest_step} and reselect the target on its image" ) } ) @@ -558,6 +576,7 @@ def _register_tools(self) -> None: } state_handlers.update( { + "view_env_state": self._view_env_state, "view_camera_meta": partial( franka_tools.view_camera_meta, state=self._state ), diff --git a/robots/dual_franka/tools.py b/robots/dual_franka/tools.py index a24e6a450..6af954018 100644 --- a/robots/dual_franka/tools.py +++ b/robots/dual_franka/tools.py @@ -580,7 +580,7 @@ def view_env_state( *, state: EnvState, ) -> ToolResult: - """Read a dual-Franka state snapshot. Configured inline camera views are returned directly; other available views are returned as artifact paths; use read_image to inspect these artifacts.""" + """Capture a fresh dual-Franka observation when step=-1 (default); a nonnegative step reads that historical snapshot without acquisition. Use the returned step for segmentation and back-projection; do not reuse pixels or masks from an older step after refreshing. Configured inline camera views are returned directly; other available views are returned as artifact paths for targeted read_image inspection.""" images: list[bytes] = [] record = state.get(step) output = record.to_blob() diff --git a/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py b/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py index 87eb374bd..596bd538e 100644 --- a/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py +++ b/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py @@ -25,7 +25,7 @@ import numpy as np import pytest -from robots.dual_franka import get_robot_spec, perception +from robots.dual_franka import get_robot_spec from robots.dual_franka.perception import ( back_project, load_calibration_bundle, @@ -643,8 +643,7 @@ def fail_json_save(name, value, **kwargs): assert "_image_" not in text_block["text"] -@pytest.mark.parametrize("parallel", [False, True]) -def test_segment_returns_mask_overlay_and_world_point(tmp_path, monkeypatch, parallel): +def test_segment_returns_mask_overlay_and_world_point(tmp_path: Path): state = EnvState(tmp_path) with state.record_step( state={ @@ -676,6 +675,7 @@ def test_segment_returns_mask_overlay_and_world_point(tmp_path, monkeypatch, par config.write_text( "perception:\n" " calibration:\n" + f" base_camera: {fixtures / 'third_to_right_base_calib_eye_on_base.yaml'}\n" f" d455_camera: {fixtures / 'd455_to_right_base_eye_on_base.yaml'}\n" " projection_views:\n" " d455:\n" @@ -690,70 +690,29 @@ def test_segment_returns_mask_overlay_and_world_point(tmp_path, monkeypatch, par " - [0.0, 0.0, 1.0, 0.0]\n" " - [0.0, 0.0, 0.0, 1.0]\n" ) - targets = ["first", "second"] if parallel else ["first"] - barrier = threading.Barrier(len(targets)) - localize = perception._mask_to_camera_world - - def synchronized_localize(*args, **kwargs): - # Both calls reach localization before either can save an artifact. - barrier.wait(timeout=3) - return localize(*args, **kwargs) - - monkeypatch.setattr(perception, "_mask_to_camera_world", synchronized_localize) - save = state.save - save_barrier = threading.Barrier(len(targets)) - - def synchronized_save(name, value, **kwargs): - if name.startswith("d455_segment_overlay_"): - save_barrier.wait(timeout=3) - return save(name, value, **kwargs) - - monkeypatch.setattr(state, "save", synchronized_save) set_robot_config_path(config) try: - call = partial( - segment, + result_native = segment( prompt="white cardboard box interior", + target_name="cardboard_box_interior", min_valid_depth_pixels=1, state=state, sam3_client=FakeSam3Client(), ) - with ThreadPoolExecutor(max_workers=len(targets)) as pool: - results = list(pool.map(lambda target: call(target_name=target), targets)) - result_native = results[0] result = result_native.data + from robots.dual_franka.perception import _mask_to_camera_world - monkeypatch.setattr(perception, "_mask_to_camera_world", localize) - failed_names = [] - - def fail_json_save(name, value, **kwargs): - if name.startswith("d455_segment_") and name.endswith(".json"): - failed_names.append(name) - return None - return save(name, value, **kwargs) - - monkeypatch.setattr(state, "save", fail_json_save) - failed = call(target_name="failed") - failed_overlay = state.load_bytes(failed.data["overlay_artifact"]) - monkeypatch.setattr(state, "save", save) - following = call(target_name="following") - assert failed.is_error - assert not following.is_error - assert following.data["segment_artifact"] not in failed_names - assert following.data["overlay_artifact"] != failed.data["overlay_artifact"] - assert state.load_bytes(failed.data["overlay_artifact"]) == failed_overlay - assert ( - state.load(following.data["segment_artifact"])["target_name"] == "following" + rejected = _mask_to_camera_world( + np.ones((8, 8), dtype=bool), + np.full((8, 8), 3.958), + camera="d455", + state=state, + step_idx=step, + min_valid=1, ) finally: set_robot_config_path(None) - assert len({item.data["segment_artifact"] for item in results}) == len(targets) - assert len({item.data["overlay_artifact"] for item in results}) == len(targets) - for target, item in zip(targets, results, strict=True): - assert not item.is_error - assert state.load(item.data["segment_artifact"])["target_name"] == target - assert item.images == [state.load_bytes(item.data["overlay_artifact"])] assert result["ok"] assert result["found"] assert result["coordinate_frame"] == "right_base" @@ -765,6 +724,12 @@ def fail_json_save(name, value, **kwargs): assert result["segment_artifact"].startswith("d455_segment_") assert result["overlay_artifact"].startswith("d455_segment_overlay_") assert result_native.images and all(result_native.images) + + assert rejected["point_xyz"] is None + assert rejected["valid_localization_pixels"] == 0 + assert rejected["raw_depth_median_m"] == 3.958 + assert rejected["mask_bbox_rc"] == [0, 0, 7, 7] + assert len(rejected["centroid_pixel"]) == 2 assert result["image_block_order"] == ["d455_segment_overlay"] blocks = mcp_result(result_native)["content"] diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index b6e14d0c7..970a01595 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -25,6 +25,7 @@ from robots.dual_franka.toolkit import DualFrankaToolkit from rpent.dashboard.events import NullDashboardEventSink from rpent.memory import MemoryManager +from rpent.tools import ToolResult class FakeEnv: @@ -686,3 +687,42 @@ def test_explore_task_is_only_in_user_prompt(): for section in ("TASK", "TASK CONSTRAINTS"): assert section not in system assert section in user + + +def test_current_view_refreshes_but_historical_view_does_not(setup, monkeypatch): + t, env, replies = setup + reset(t, replies) + old_step = t.state.latest_step + reads = [] + published = [] + monkeypatch.setattr(t, "_publish_step", published.append) + original = env.get_observation + + def observe(): + reads.append(True) + obs = original() + obs["d455_images"][:] = 77 + return obs + + env.get_observation = observe + observed = t.execute_tool("view_env_state", {}) + assert len(reads) == 1 and t.state.latest_step == old_step + 1 + assert observed.images == [t.state.load_bytes("d455.png")] + assert len(published) == 1 + assert published[0].command == {"action": "observe_current"} + assert np.all(t.state.load("d455.png") == 77) + call(t, "view_env_state", step=old_step) + assert len(reads) == 1 + assert ( + "error" in t._current_perception(lambda **kw: ToolResult(), step=old_step).data + ) + t._current_perception(lambda **kw: ToolResult(), step=t.state.latest_step) + assert len(reads) == 1 # Localization must not silently acquire another frame. + assert len(published) == 1 + assert env.resets == 1 and not env.moves + call(t, "move_delta", arm="left", delta_xyz=[0.01, 0, 0]) + assert len(reads) == 2 and t.state.latest_step == old_step + 2 + assert len(published) == 2 + assert published[-1].command["action"] == "move_delta" + assert len(env.moves) == 1 and env.moves[0][0] == "left" + np.testing.assert_allclose(env.moves[0][1], [0.01, 0, 0]) diff --git a/tests/unit_tests/robots/fixtures/tool_schemas.json b/tests/unit_tests/robots/fixtures/tool_schemas.json index 7a1bb3176..a4e13de10 100644 --- a/tests/unit_tests/robots/fixtures/tool_schemas.json +++ b/tests/unit_tests/robots/fixtures/tool_schemas.json @@ -940,7 +940,7 @@ }, { "name": "view_env_state", - "description": "Read a dual-Franka state snapshot. Configured inline camera views are returned directly; other available views are returned as artifact paths; use read_image to inspect these artifacts.", + "description": "Capture a fresh dual-Franka observation when step=-1 (default); a nonnegative step reads that historical snapshot without acquisition. Use the returned step for segmentation and back-projection; do not reuse pixels or masks from an older step after refreshing. Configured inline camera views are returned directly; other available views are returned as artifact paths for targeted read_image inspection.", "input_schema": { "type": "object", "properties": { From 8e12a26eb74441f5ccc24a0101c9666fc4d5a970 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 21:13:43 +0800 Subject: [PATCH 02/18] fix(dual-franka): dashboard-operator review slice --- robots/dual_franka/robot_spec.py | 7 +- rpent/cli/dashboard.py | 70 +++- rpent/cli/main.py | 21 +- rpent/dashboard/operator.py | 144 +++++++++ rpent/dashboard/server.py | 5 +- rpent/dashboard/state.py | 108 ++++++- rpent/dashboard/static/interaction.js | 17 +- .../dual_franka/test_dashboard_operator.py | 305 ++++++++++++++++++ .../rpent/dashboard/test_server_contracts.py | 38 +++ .../rpent/dashboard/test_session_contracts.py | 115 ++++++- .../rpent/dashboard/test_state_contracts.py | 71 ++++ 11 files changed, 872 insertions(+), 29 deletions(-) create mode 100644 rpent/dashboard/operator.py create mode 100644 tests/unit_tests/robots/dual_franka/test_dashboard_operator.py diff --git a/robots/dual_franka/robot_spec.py b/robots/dual_franka/robot_spec.py index aa9dbffc4..6705427b1 100644 --- a/robots/dual_franka/robot_spec.py +++ b/robots/dual_franka/robot_spec.py @@ -56,7 +56,7 @@ "name": "task_id", "kind": "integer", "minimum": 0, - "suggestions": tuple(sorted(DUAL_FRANKA_TASKS)), + "suggestions": tuple(str(key) for key in sorted(DUAL_FRANKA_TASKS)), }, ), "display": "Dual Franka task {task_id}", @@ -424,7 +424,10 @@ def _init_runtime( connectors = { "env": lambda rpc: { "env": DualFrankaEnvClient( - rpc, reset_on_connect=not getattr(args, "explore", False) + rpc, + reset_on_connect=not ( + getattr(args, "explore", False) or getattr(args, "dashboard", False) + ), ), "task_description": get_dual_franka_task(args.task_id).instruction, "vla_instruction": get_dual_franka_task(args.task_id).vla_instruction, diff --git a/rpent/cli/dashboard.py b/rpent/cli/dashboard.py index 072e256ff..b1ed947d2 100644 --- a/rpent/cli/dashboard.py +++ b/rpent/cli/dashboard.py @@ -33,6 +33,7 @@ ) from rpent.dashboard.events import RunStartedEvent from rpent.evaluation import RunFinalizationContext +from rpent.evaluation.result import record_operator_outcome from rpent.memory import MemoryManager from rpent.planner.base import build_planner from rpent.robots import get_toolkit @@ -54,10 +55,17 @@ def run_dashboard_session( parser: argparse.ArgumentParser, ) -> int: """Run one long-lived Dashboard Session with sequential fresh TaskRuns.""" - if robot_spec.is_real_robot and not (robot_spec.dashboard or {}).get( - "external_env", False + if ( + robot_spec.is_real_robot + and not (robot_spec.dashboard or {}).get("external_env", False) + and ( + not robot_spec.supports_human_interactive_exploration + or args.planner != "codex" + ) ): - parser.error("This robot requires operator confirmation in a plain terminal.") + parser.error( + "Physical Dashboard control requires an attended Codex robot runtime." + ) from rpent.dashboard.server import DashboardServer from rpent.dashboard.session import DashboardSessionController from rpent.dashboard.state import DashboardState @@ -83,8 +91,12 @@ def run_dashboard_session( if component["scope"] == "unique" } - if getattr(args, "env_endpoint", None) is not None and not dashboard_spec.get( - "external_env", False + if getattr(args, "env_endpoint", None) is not None and not ( + dashboard_spec.get("external_env", False) + or ( + robot_spec.is_real_robot + and robot_spec.supports_human_interactive_exploration + ) ): parser.error( "Dashboard task control cannot use --env-endpoint because each " @@ -178,6 +190,9 @@ def _run_dashboard_task( session_root: Path, ) -> str | None: """Execute one fresh Dashboard TaskRun against Session-owned services.""" + human_interactive_run = ( + robot_spec.is_real_robot and robot_spec.supports_human_interactive_exploration + ) task_args = copy.copy(args) for name, value in claimed.request.items(): setattr(task_args, name, value) @@ -215,6 +230,13 @@ def _run_dashboard_task( } prompt_vars = {**run_config.prompt_vars, "output_dir": output_dir} session_message = robot_spec.prompts.render("user", variables=prompt_vars) + if human_interactive_run: + session_message += ( + "\nDASHBOARD OPERATOR: Before the first motion call request_scene_reset " + "and wait for the human's web confirmation. Final success/failure " + "requires request_operator_verdict or an explicit operator verdict. " + "There is no terminal input; do not ask for shell/TTY replies." + ) sessions = max( 1, int(getattr(task_args, "explore_sessions", 1) or 1), @@ -255,6 +277,14 @@ def _run_dashboard_task( video_path=state_output_dir / "episode.mp4", ) if robot_spec.supports_exploration: + operator_kwargs = {} + if human_interactive_run: + from rpent.dashboard.operator import DashboardOperator + + state.operator = DashboardOperator( + state_output_dir, on_change=state.operator_changed + ) + operator_kwargs["operator_input"] = state.operator toolkit = get_toolkit( args.robot_name, runtime_kwargs=runtime_kwargs, @@ -265,6 +295,7 @@ def _run_dashboard_task( task_args, "explore_attempts_per_session", 0 ), state_output_dir=state_output_dir, + **operator_kwargs, ) else: toolkit = get_toolkit( @@ -303,6 +334,22 @@ def _run_dashboard_task( messages += result.messages stats = result.stats agent_error = result.error + if human_interactive_run and toolkit.direct_verdict_requested: + finish_result = toolkit.finalize_direct_verdict() + if task_args.explore and finish_result["operator_verdict"] in { + "success", + "failure", + }: + record_operator_outcome( + memory_root=memory_manager.root, + recipe_tag=recipe_tag, + run_name=output_dir.name, + session_number=session_number, + state_output_dir=state_output_dir, + verdict=finish_result, + messages=_serialize_messages(result.messages), + planner_error=agent_error, + ) if robot_spec.supports_exploration: solved = bool(toolkit.solved()) if solved: @@ -375,8 +422,12 @@ def _run_dashboard_task( getattr(task_args, "explore", False) and getattr(task_args, "auto_merge_memory", False) and not agent_error - and not state.task_replacement_requested + and ( + not state.task_replacement_requested + or (human_interactive_run and state.operator_completed_task) + ) and memory_manager is not None + and (not human_interactive_run or solved) ): try: merge_result = memory_manager.merge_memory( @@ -420,4 +471,11 @@ def _run_dashboard_task( logger.warning("%s", finalization_error) else: agent_error = finalization_error + if ( + human_interactive_run + and not solved + and not agent_error + and not state.task_replacement_requested + ): + return "Task ended without operator-confirmed success" return agent_error diff --git a/rpent/cli/main.py b/rpent/cli/main.py index 9913c35f7..120e77cb1 100644 --- a/rpent/cli/main.py +++ b/rpent/cli/main.py @@ -343,13 +343,16 @@ def main() -> int: human_interactive_exploration = ( args.explore and robot_spec.supports_human_interactive_exploration ) + human_interactive_run = human_interactive_exploration or ( + args.dashboard and robot_spec.supports_human_interactive_exploration + ) if args.dashboard and args.interactive: parser.error("--dashboard and --interactive cannot be used together") external_env_dashboard = (getattr(robot_spec, "dashboard", None) or {}).get( "external_env", False ) if robot_spec.is_real_robot and not ( - human_interactive_exploration or external_env_dashboard + human_interactive_run or external_env_dashboard ): if args.dashboard or args.interactive: parser.error( @@ -382,11 +385,7 @@ def main() -> int: ) if args.explore and getattr(args, "explore_attempts_per_session", 0) < 0: parser.error("--explore-attempts-per-session must be nonnegative") - if human_interactive_exploration: - if args.dashboard: - parser.error( - "Human-interactive exploration currently requires the CLI operator terminal; Dashboard feedback is not implemented" - ) + if human_interactive_run and not args.dashboard: if sys.stdin is None or not sys.stdin.isatty(): parser.error( "Human-interactive exploration requires a TTY for operator reset/verdict feedback" @@ -457,7 +456,7 @@ def main() -> int: ) operator_input = None - if human_interactive_exploration: + if human_interactive_run: from rpent.tools.human_in_the_loop import HumanInTheLoopInput operator_input = HumanInTheLoopInput( @@ -511,7 +510,7 @@ def main() -> int: first_user_msg = await_first_prompt() if first_user_msg is None: logger.info("no task entered; ending session before start.") - if human_interactive_exploration: + if human_interactive_run: prompt_vars = {**prompt_vars, "initial_user_message": first_user_msg} # Exploration may hand off between independent planner contexts. sessions = max(1, int(getattr(args, "explore_sessions", 1) or 1)) @@ -600,7 +599,7 @@ def accept_verdict( pending_operator_request = None try: solve = planner.solve - if human_interactive_exploration and input_queue is not None: + if human_interactive_run and input_queue is not None: from functools import partial solve = partial( @@ -623,7 +622,7 @@ def accept_verdict( agent_error = result.error if operator_input is not None and args.interactive: pending_operator_request = operator_input.pending_kind - if human_interactive_exploration and toolkit.direct_verdict_requested: + if human_interactive_run and toolkit.direct_verdict_requested: finish_result, agent_error, verdict_handoff = ( _finalize_operator_verdict( result, @@ -673,7 +672,7 @@ def accept_verdict( if operator_input is not None and args.interactive: operator_input.cancel_pending() if ( - human_interactive_exploration + human_interactive_run and not continue_after_verdict and not (finish_result or {}).get("_finish") and not (finish_result or {}).get("operator_finished") diff --git a/rpent/dashboard/operator.py b/rpent/dashboard/operator.py new file mode 100644 index 000000000..a066b9b5c --- /dev/null +++ b/rpent/dashboard/operator.py @@ -0,0 +1,144 @@ +# Copyright 2026 The RPent Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cancelable, request-scoped operator confirmations for physical tasks.""" + +from __future__ import annotations + +import json +import threading +import uuid +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from rpent.tools.toolkit import ToolCancelled + + +class DashboardOperator: + """Own one pending confirmation and archive its request and reply.""" + + def __init__( + self, output_dir: str | Path, on_change: Callable[[], None] = lambda: None + ) -> None: + """Initialize the confirmation channel. + + Args: + output_dir: Directory for the operator event log. + on_change: Callback invoked when a request opens or closes. + """ + self._condition = threading.Condition() + self._pending: dict[str, Any] | None = None + self._answer: str | None = None + self._closed = False + self._path = Path(output_dir) / "dashboard_operator.jsonl" + self._on_change = on_change + + def _record(self, event: dict[str, Any]) -> None: + self._path.parent.mkdir(parents=True, exist_ok=True) + with self._path.open("a") as file: + file.write(json.dumps(event, ensure_ascii=False) + "\n") + + def snapshot(self) -> dict[str, Any] | None: + """Return the pending request, or None when no confirmation is pending.""" + with self._condition: + return dict(self._pending) if self._pending else None + + def request( + self, + prompt: str, + check_cancelled: Callable[[], None], + *, + kind: str | None = None, + ) -> str: + """Wait for an operator reply while checking cancellation. + + Args: + prompt: Instructions shown to the operator. + check_cancelled: Callback that raises when the operation is cancelled. + kind: ``reset`` permits done/abort; other kinds permit task verdicts. + + Returns: + The accepted answer followed by any operator notes. + + Raises: + ToolCancelled: The channel closes or the callback reports cancellation. + RuntimeError: Another confirmation is already pending. + """ + choices = ( + ["done", "abort"] + if kind == "reset" + else ["success", "failure", "continue", "abort"] + ) + with self._condition: + if self._closed: + raise ToolCancelled("operator channel closed") + if self._pending: + raise RuntimeError("another operator confirmation is pending") + self._pending = { + "id": uuid.uuid4().hex, + "kind": kind, + "prompt": prompt, + "choices": choices, + } + self._answer = None + self._record({"event": "request", **self._pending}) + self._on_change() + try: + with self._condition: + while self._answer is None: + check_cancelled() + if self._closed: + raise ToolCancelled("operator channel closed") + self._condition.wait(0.1) + check_cancelled() + return self._answer + finally: + with self._condition: + self._pending = None + self._answer = None + self._on_change() + + def reply(self, request_id: str, answer: str, notes: str = "") -> None: + """Answer the matching pending request once. + + Args: + request_id: Identifier from the pending request snapshot. + answer: One of the choices exposed by that request. + notes: Optional instructions or evidence accompanying the answer. + + Raises: + ValueError: The request is expired, answered, or the reply is invalid. + """ + with self._condition: + if ( + self._closed + or not self._pending + or request_id != self._pending["id"] + or self._answer is not None + ): + raise ValueError("expired or already answered operator request") + if answer not in self._pending["choices"] or not isinstance(notes, str): + raise ValueError("invalid operator response") + self._answer = answer + (" " + notes.strip() if notes.strip() else "") + self._record( + {"event": "reply", "id": request_id, "answer": answer, "notes": notes} + ) + self._condition.notify_all() + + def close(self) -> None: + """Close the channel and wake waiting callers without confirming anything.""" + with self._condition: + self._closed = True + self._condition.notify_all() diff --git a/rpent/dashboard/server.py b/rpent/dashboard/server.py index 7c9549411..7e67dd006 100644 --- a/rpent/dashboard/server.py +++ b/rpent/dashboard/server.py @@ -187,7 +187,10 @@ def api_submit_message( payload: dict[str, Any] = Body(default={}), ) -> JSONResponse: try: - self._state.submit_input(payload.get("text")) + self._state.submit_input( + payload.get("text"), + operator_context=payload.get("operator_context"), + ) except ValueError as exc: return JSONResponse({"error": str(exc)}, status_code=422) except InteractionUnavailableError as exc: diff --git a/rpent/dashboard/state.py b/rpent/dashboard/state.py index 4c26f98b7..3d4148a7c 100644 --- a/rpent/dashboard/state.py +++ b/rpent/dashboard/state.py @@ -45,6 +45,7 @@ UnknownDashboardMessageError, ) from rpent.dashboard.spec import DashboardSpec, TaskSpec +from rpent.tools.human_in_the_loop import parse_operator_command from rpent.utils.logging import get_logger from rpent.utils.templates import substitute @@ -193,7 +194,7 @@ def __init__( } self._primitive_allowlist = dashboard_spec["primitives"] - self._lock = threading.Lock() + self._lock = threading.RLock() self._condition = threading.Condition(self._lock) self._next_model: str | None = None self._task_state: str | None = None @@ -235,6 +236,54 @@ def __init__( self._manual_results: list[str] = [] self._explore_paused = False self._exploration_message: DashboardMessage | None = None + self.operator = None + self.operator_verdict = None + + def operator_changed(self): + with self._condition: + self._projection_changed_locked() + + def operator_snapshot(self): + with self._condition: + return { + "enabled": self.operator is not None, + "active": self._toolkit is not None, + "generation": self._task_generation, + "pending": self.operator.snapshot() if self.operator else None, + "verdict": self.operator_verdict, + } + + def operator_reply(self, generation, request_id, answer, notes=""): + with self._condition: + if ( + generation != self._task_generation + or self.operator is None + or self._session_state == "switch_pending" + ): + raise ValueError("expired operator task") + self.operator.reply(request_id, answer, notes) + + def operator_finish(self, generation, verdict, notes=""): + with self._condition: + if verdict not in {"success", "failure", "abort"} or not isinstance( + notes, str + ): + raise ValueError("invalid operator verdict") + if ( + generation != self._task_generation + or self.operator is None + or self._toolkit is None + or self._session_state == "switch_pending" + ): + raise ValueError("no matching active operator task") + if not self._toolkit.request_direct_verdict(verdict, notes): + raise ValueError("operator verdict refused in current state") + self.operator_verdict = verdict + self._pending_task = None + self._session_state = "switch_pending" + self._accepting_input = False + self.operator.close() + self._interaction_changed_locked() def bind_toolkit(self, toolkit: Toolkit) -> None: """Expose a TaskRun Toolkit through the Dashboard primitive API.""" @@ -250,6 +299,8 @@ def unbind_toolkit(self, toolkit: Toolkit) -> None: with self._condition: if self._toolkit is toolkit: self._toolkit = None + if self.operator is not None: + self.operator.close() self._projection_changed_locked() toolkit.cancel_active_and_wait() with self._condition: @@ -394,6 +445,16 @@ def task_replacement_requested(self) -> bool: with self._lock: return self._session_state == "switch_pending" + @property + def operator_completed_task(self) -> bool: + """Whether operator completion is draining without a queued replacement.""" + with self._lock: + return ( + self._session_state == "switch_pending" + and self.operator_verdict is not None + and self._pending_task is None + ) + def shared_services_ready(self) -> None: """Open the command channel after shared services have started.""" with self._condition: @@ -475,7 +536,9 @@ def continue_exploration(self, *, explicit: bool = False) -> None: self._active_primitive_calls.pop(key, None) self._interaction_changed_locked() - def submit_input(self, text: str) -> DashboardMessage | TaskRequest: + def submit_input( + self, text: str, *, operator_context=None + ) -> DashboardMessage | TaskRequest | None: """Route a local task command or a normal conversation message.""" if not isinstance(text, str) or not text.strip(): return self._submit_message(text) @@ -486,6 +549,38 @@ def submit_input(self, text: str) -> DashboardMessage | TaskRequest: raise ValueError(guidance) self.continue_exploration(explicit=True) return {"command": command} + with self._condition: + parsed = parse_operator_command(text) if self.operator is not None else None + if parsed is not None: + command = f"/{parsed.name}" + notes = parsed.notes + current = self.operator_snapshot() + if operator_context is None: + raise ValueError( + "No active operator channel; refresh the Dashboard." + ) + if not isinstance(operator_context, dict): + raise ValueError("operator_context must be an object.") + if operator_context.get("generation") != current["generation"]: + raise ValueError("expired operator task") + pending = current["pending"] + if command == "/continue" and pending is None: + return self._submit_message(text) + if command in {"/success", "/failure", "/abort"}: + self.operator_finish(current["generation"], command[1:], notes) + else: + request_id = operator_context.get("request_id") + answer = command[1:] + if command == "/operator": + request_id, answer = parsed.request_id, parsed.answer + if pending is None or request_id != pending["id"]: + raise ValueError("No matching pending operator request.") + self.operator_reply( + current["generation"], request_id, answer, notes + ) + self._control_feedback = [f"{command} accepted"] + self._interaction_changed_locked() + return None try: request = _parse_task(self._task_spec, text) except ValueError as exc: @@ -591,6 +686,12 @@ def complete_task( if state not in TERMINAL_RUN_STATES: raise ValueError(f"invalid terminal run state: {state!r}") with self._condition: + if self.operator_verdict is not None and error is None: + state = { + "success": "succeeded", + "failure": "failed", + "abort": "cancelled", + }[self.operator_verdict] self._task_state = state self._error = None if error is None else str(error) self._seal_interaction_locked() @@ -609,6 +710,8 @@ def _begin_task_locked( output_dir: Path, ) -> None: self._current_task = request + self.operator = None + self.operator_verdict = None self.video_path = output_dir / "episode.mp4" self._session_state = "task_starting" self._task_state = "starting" @@ -1083,6 +1186,7 @@ def _session_fields_locked(self) -> dict[str, Any]: "pending_task": self._command_snapshot(self._pending_task), "control_feedback": list(self._control_feedback), "control_error": self._control_error, + "operator": self.operator_snapshot(), } def _visible_state_locked(self) -> str: diff --git a/rpent/dashboard/static/interaction.js b/rpent/dashboard/static/interaction.js index 1382d9eec..166cd68a7 100644 --- a/rpent/dashboard/static/interaction.js +++ b/rpent/dashboard/static/interaction.js @@ -249,6 +249,10 @@ export function createInteractionController({ copy, select }) { const backendError = errorText(interaction.last_error); const controlError = errorText(session.control_error); const feedback = session.control_feedback.map(errorText).filter(Boolean); + const operator = session.operator; + if (operator?.enabled) { + select("#chatHint").textContent += " · /done · /continue · /success · /failure · /abort"; + } if (state.requestError) { status.textContent = state.requestError; status.classList.add("is-error"); @@ -267,6 +271,11 @@ export function createInteractionController({ copy, select }) { } else if (interaction.interrupt_requested || state.interruptInFlight) { status.textContent = copy.interruptRequested; status.classList.add("is-busy"); + } else if (operator?.pending) { + status.textContent = operator.pending.prompt + (operator.pending.kind === "reset" + ? "\nEnter /done to confirm scene restoration and allow the robot to reset, or /abort to cancel." + : "\nEnter /continue, /success, /failure or /abort, optionally followed by notes."); + status.classList.add("is-ready"); } else if (state.notice) { status.textContent = state.notice; status.classList.add("is-ready"); @@ -380,7 +389,13 @@ export function createInteractionController({ copy, select }) { try { await requestJSON( "/api/session/messages", - { method: "POST", body: { text } }, + { method: "POST", body: { + text, + operator_context: { + generation: state.snapshot.task_generation, + request_id: state.snapshot.operator?.pending?.id ?? null, + }, + } }, ); if (input.value === draft) input.value = ""; } catch (error) { diff --git a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py new file mode 100644 index 000000000..2bde6997d --- /dev/null +++ b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py @@ -0,0 +1,305 @@ +# Copyright 2026 The RPent Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Web operator contracts using real Dashboard, toolkit and memory components.""" + +import json +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from types import SimpleNamespace + +import pytest +from fastapi.testclient import TestClient + +from robots.dual_franka import robot_spec +from robots.dual_franka.toolkit import DualFrankaToolkit +from rpent.dashboard.events import NullDashboardEventSink +from rpent.dashboard.operator import DashboardOperator +from rpent.dashboard.server import DashboardServer +from rpent.dashboard.state import DashboardState +from rpent.memory import MemoryManager +from rpent.planner.base import PlannerResult +from tests.unit_tests.robots.dual_franka.test_exploration import FakeEnv + + +def call(toolkit, name, **kwargs): + return toolkit.execute_tool(name, kwargs).data + + +def wait_for_request(operator): + deadline = time.monotonic() + 3 + while operator.snapshot() is None and time.monotonic() < deadline: + time.sleep(0.01) + assert operator.snapshot() is not None + + +@pytest.fixture +def pending_reset(tmp_path, monkeypatch, dual_franka_robot_config): + from robots.franka.runtime_config import set_robot_config_path + + set_robot_config_path(dual_franka_robot_config) + monkeypatch.setattr("robots.dual_franka.toolkit.get_output_dir", lambda: tmp_path) + env = FakeEnv() + state = DashboardState( + output_dir=tmp_path, dashboard_spec=robot_spec.DUAL_FRANKA_DASHBOARD_SPEC + ) + state.shared_services_ready() + state.request_task({"task_id": 3}) + state.wait_for_task(0) + operator = DashboardOperator(tmp_path) + state.operator = operator + toolkit = DualFrankaToolkit( + runtime_kwargs={"env": env, "model": None, "task_description": "test"}, + dashboard_events=NullDashboardEventSink(), + memory=MemoryManager( + tmp_path / "memory", memory_access="inbox_write", inbox_cell_tag="test" + ), + mode="evaluation", + state_output_dir=tmp_path, + operator_input=state.operator, + ) + state.bind_toolkit(toolkit) + with ( + TestClient(DashboardServer(state=state)._app) as client, + ThreadPoolExecutor() as pool, + ): + future = pool.submit(call, toolkit, "request_scene_reset", reason="web reset") + try: + wait_for_request(state.operator) + pending = client.get("/api/session/state").json()["operator"] + yield toolkit, env, state, client, pending, future + finally: + operator.close() + toolkit.close() + state.unbind_toolkit(toolkit) + + +def message(pending, text): + return { + "text": text, + "operator_context": { + "generation": pending["generation"], + "request_id": pending["pending"]["id"], + }, + } + + +@pytest.mark.parametrize("answer", ["done", "abort"]) +def test_reset_waits_for_one_matching_reply(pending_reset, answer): + toolkit, env, _, client, pending, future = pending_reset + reply = message(pending, f"/{answer} scene checked") + assert env.resets == 0 + assert client.post("/api/session/messages", json=reply).status_code == 202 + result = future.result(3) + assert env.resets == int(answer == "done") + if answer == "done": + assert toolkit.state.latest_record().result["operator_notes"] == "scene checked" + else: + assert result["code"] == "tool_cancelled" + assert client.post("/api/session/messages", json=reply).status_code == 422 + assert env.resets == int(answer == "done") + + +@pytest.mark.parametrize( + "context", + [ + {"generation": -1}, + {"generation": 1, "request_id": "stale"}, + [], + ], +) +def test_invalid_reply_cannot_release_reset(pending_reset, context): + _, env, _, client, pending, future = pending_reset + reply = {**message(pending, "/done"), "operator_context": context} + assert client.post("/api/session/messages", json=reply).status_code == 422 + assert not future.done() and env.resets == 0 + + +@pytest.mark.parametrize("verdict", ["success", "failure", "abort"]) +def test_verdict_closes_only_current_task(pending_reset, verdict): + toolkit, env, state, client, pending, future = pending_reset + assert ( + client.post("/api/session/messages", json=message(pending, "/done")).status_code + == 202 + ) + future.result(3) + reply = message(pending, f"/{verdict} checked") + assert client.post("/api/session/messages", json=reply).status_code == 202 + assert call(toolkit, "open_gripper", arm="right")["motion_refused"] + toolkit.finalize_direct_verdict() + assert toolkit.solved() is (verdict == "success") + assert client.post("/api/session/messages", json=reply).status_code == 422 + state.unbind_toolkit(toolkit) + state.complete_task(state="cancelled") + assert ( + state._task_state + == { + "success": "succeeded", + "failure": "failed", + "abort": "cancelled", + }[verdict] + ) + state.request_task({"task_id": 4}) + state.wait_for_task(0) + state.bind_toolkit(toolkit) + assert client.post("/api/session/messages", json=reply).status_code == 409 + assert env.resets == 1 + + +def test_late_verdict_preserves_queued_task(pending_reset): + toolkit, _, state, client, pending, _ = pending_reset + assert ( + client.post( + "/api/session/messages", json=message(pending, "/rpent-task 4") + ).status_code + == 202 + ) + assert ( + client.post( + "/api/session/messages", json=message(pending, "/success") + ).status_code + == 422 + ) + assert not toolkit.direct_verdict_requested + state.unbind_toolkit(toolkit) + state.complete_task(state="cancelled") + assert state.wait_for_task(0).request == {"task_id": 4} + + +def test_cancel_releases_operator_wait_without_reset(pending_reset): + toolkit, env, state, _, _, future = pending_reset + toolkit.cancel_active_and_wait() + future.result(3) + assert env.resets == 0 and state.operator.snapshot() is None + + +@pytest.mark.parametrize( + "explore,verdict,auto_merge,error,replacement", + [ + (False, "success", True, None, False), + (True, "success", True, None, False), + (True, "success", False, None, False), + (True, "success", True, "planner failed", False), + (True, "failure", True, None, False), + (True, "abort", True, None, False), + (True, None, True, None, True), + (True, "success", True, None, True), + ], +) +def test_task_uses_web_operator_and_publishes_only_confirmed_success( + tmp_path, + monkeypatch, + dual_franka_robot_config, + explore, + verdict, + auto_merge, + error, + replacement, +): + from rpent.cli import dashboard, main + + parser = main._build_argparser() + robot_spec._add_cli_args(parser, use_dashboard=True) + args = parser.parse_args( + [ + "--robot", + "dual_franka", + "--dashboard", + "--task-id", + "3", + "--robot-config", + str(dual_franka_robot_config), + "--planner", + "codex", + "--memory-profile", + "local", + "--memory-dir", + str(tmp_path / "memory"), + "--auto-merge-memory" if auto_merge else "--no-auto-merge-memory", + ] + ) + args.explore = explore + env = FakeEnv() + spec = replace( + robot_spec.get_robot_spec(), + init_runtime=lambda *args: ( + [], + {"env": env, "model": None, "task_description": "test"}, + ), + ) + state = DashboardState(output_dir=tmp_path, dashboard_spec=spec.dashboard) + state.shared_services_ready() + state.request_task({"task_id": 3}) + claimed = state.wait_for_task(0) + + def solve(**kwargs): + toolkit = kwargs["toolkit"] + assert kwargs["dashboard_interaction"] is state + assert env.resets == 0 + assert "error" in call( + toolkit, "move_delta", arm="right", delta_xyz=[0, 0, 0.01] + ) + with ThreadPoolExecutor() as pool: + future = pool.submit(call, toolkit, "request_scene_reset", reason="test") + try: + wait_for_request(state.operator) + pending = state.operator_snapshot() + state.operator_reply( + pending["generation"], pending["pending"]["id"], "done" + ) + future.result(3) + finally: + state.operator.close() + if verdict is not None: + state.operator_finish(1, verdict, "checked") + if replacement: + state.request_task({"task_id": 4}) + return PlannerResult(error=error) + + monkeypatch.setattr( + dashboard, "build_planner", lambda *a, **kw: SimpleNamespace(solve=solve) + ) + assert ( + dashboard._run_dashboard_task( + args=args, + robot_spec=spec, + state=state, + claimed=claimed, + shared_runtime_kwargs={}, + unique_components={"env"}, + session_root=tmp_path, + ) + == error + ) + assert env.resets == 1 + published = ( + explore + and verdict == "success" + and auto_merge + and error is None + and not replacement + ) + for filename in ("dual_franka_t3.json", "dual_franka_t3_recipe.jsonl"): + assert (tmp_path / "memory/task-specific" / filename).is_file() is published + if explore and verdict in {"success", "failure"}: + evidence = list((tmp_path / "memory").rglob(f"*-{verdict}.json")) + assert len(evidence) == 1 + assert ( + json.loads(evidence[0].read_text())["verdict"]["operator_verdict"] + == verdict + ) + state.complete_task(state="cancelled", error=error) + if replacement: + assert state.wait_for_task(0).request == {"task_id": 4} diff --git a/tests/unit_tests/rpent/dashboard/test_server_contracts.py b/tests/unit_tests/rpent/dashboard/test_server_contracts.py index 340def908..b2e12c53d 100644 --- a/tests/unit_tests/rpent/dashboard/test_server_contracts.py +++ b/tests/unit_tests/rpent/dashboard/test_server_contracts.py @@ -242,3 +242,41 @@ def test_healthz_still_reports_transport_liveness_only(state: DashboardState) -> assert resp.status_code == 200 assert resp.json() == {"ok": True} + + +@pytest.mark.parametrize( + "text", + [ + "/done", + "/continue keep the completed half", + "/success", + "/failure missed grasp", + "/abort", + "/operator", + "/operator previous-request done", + ], +) +@pytest.mark.parametrize( + "operator_context", + [None, {"generation": -1, "request_id": "previous-request"}], +) +def test_generic_messages_do_not_enable_operator_control(state, text, operator_context): + state.shared_services_ready() + state.request_task({"seed": 1}) + state.wait_for_task(timeout=0) + state.set_planner_activity("idle", accepting_input=True) + client = _client(_server(state)) + + response = client.post( + "/api/session/messages", + json={"text": text, "operator_context": operator_context}, + ) + + assert response.status_code == 202 + assert state.claim_next_pending_message().text == text + assert state.claim_next_pending_message() is None + snapshot = client.get("/api/session/state").json() + assert snapshot["operator"]["enabled"] is False + assert snapshot["operator"]["verdict"] is None + assert snapshot["task_generation"] == 1 + assert not state.task_replacement_requested diff --git a/tests/unit_tests/rpent/dashboard/test_session_contracts.py b/tests/unit_tests/rpent/dashboard/test_session_contracts.py index 5aa1ef2a3..ce2efc31f 100644 --- a/tests/unit_tests/rpent/dashboard/test_session_contracts.py +++ b/tests/unit_tests/rpent/dashboard/test_session_contracts.py @@ -175,11 +175,13 @@ def test_dashboard_session_stops_shared_daemons_in_reverse_after_cleanup_error( @pytest.mark.parametrize("merge_fails", [False, True]) @pytest.mark.parametrize("sessions", [1, 2]) +@pytest.mark.parametrize("real_robot", [False, True]) def test_dashboard_exploration_finalizes_memory_and_reports_merge_failures( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, merge_fails: bool, sessions: int, + real_robot: bool, ) -> None: from rpent.cli import dashboard as dashboard_cli @@ -195,6 +197,7 @@ def merge_memory(self, **kwargs: Any) -> dict[str, int]: class FakeToolkit: memory = FakeMemoryManager() + direct_verdict_requested = False def cancel_active_and_wait(self) -> None: pass @@ -233,6 +236,9 @@ def unbind_toolkit(self, toolkit: Any) -> None: def report_task_warning(self, warning: str) -> None: self.warnings.append(warning) + def operator_changed(self) -> None: + pass + class FakePlanner: def solve(self, **kwargs: Any) -> PlannerResult: del kwargs @@ -254,9 +260,13 @@ def solve(self, **kwargs: Any) -> PlannerResult: finalize_run=None, prepare_memory=None, supports_exploration=True, - is_real_robot=False, + is_real_robot=real_robot, + supports_human_interactive_exploration=real_robot, parse_config=lambda args: run_config, - init_runtime=lambda *args: ([], {}), + init_runtime=lambda *args: ( + [], + {"options": "current-task", "task_description": "current-task"}, + ), prompts=PromptBundle( system=lambda variables: "system", user=lambda variables: "user", @@ -284,9 +294,16 @@ def solve(self, **kwargs: Any) -> PlannerResult: from rpent.cli import main as main_cli monkeypatch.setattr(main_cli, "get_robot_spec", lambda name: robot_spec) - monkeypatch.setattr( - dashboard_cli, "get_toolkit", lambda *args, **kwargs: FakeToolkit() - ) + + def get_toolkit(*args, **kwargs): + assert kwargs["runtime_kwargs"]["options"] == "shared-default" + assert kwargs["runtime_kwargs"]["task_description"] == "current-task" + assert kwargs["runtime_kwargs"]["shared_client"] == "client" + if real_robot: + assert kwargs["operator_input"] is state.operator + return FakeToolkit() + + monkeypatch.setattr(dashboard_cli, "get_toolkit", get_toolkit) monkeypatch.setattr( dashboard_cli, "build_planner", lambda *args, **kwargs: FakePlanner() ) @@ -296,7 +313,7 @@ def solve(self, **kwargs: Any) -> PlannerResult: robot_spec=robot_spec, state=state, claimed=claimed, - shared_runtime_kwargs={}, + shared_runtime_kwargs={"options": "shared-default", "shared_client": "client"}, unique_components=set(), session_root=tmp_path / "session", ) @@ -318,3 +335,89 @@ def solve(self, **kwargs: Any) -> PlannerResult: ) else: assert state.warnings == [] + + +@pytest.mark.parametrize("environment_success", [False, True]) +def test_external_robot_dashboard_keeps_its_own_operator_and_result_contract( + tmp_path, monkeypatch, environment_success +): + from robots.yam.robot_spec import YAM_DASHBOARD_SPEC, get_robot_spec + from rpent.cli import dashboard as dashboard_cli + from rpent.dashboard.state import DashboardState + + config = RunConfig( + recipe_tag="yam_t0", + output_dir=tmp_path / "task", + prompt_vars={}, + task_desc={"robot": "yam"}, + ) + finalized = [] + closed = [] + toolkit = SimpleNamespace( + memory=None, + solved=lambda: environment_success, + write_recipe=lambda recipe_tag: "", + cancel_active_and_wait=lambda: None, + close=lambda: closed.append(True), + ) + + def get_toolkit(*args, **kwargs): + assert "operator_input" not in kwargs + assert kwargs["mode"] == "evaluation" + return toolkit + + def solve(**kwargs): + assert "DASHBOARD OPERATOR" not in kwargs["user_message"] + return PlannerResult(finish_result={"success": True}, messages=[], stats={}) + + from dataclasses import replace + + spec = replace( + get_robot_spec(), + parse_config=lambda args: config, + init_runtime=lambda *args: ([], {}), + prepare_memory=lambda args, run_config: None, + prompts=PromptBundle( + system=lambda variables: "system", user=lambda variables: "user" + ), + finalize_run=finalized.append, + ) + args = SimpleNamespace( + verbose=False, + robot_name="yam", + explore=False, + auto_merge_memory=False, + planner="api", + base_url=None, + model="offline", + max_tokens=128, + planner_timeout_s=10, + reasoning_effort=None, + claude_code_max_budget_usd=None, + no_images=False, + max_turns=2, + ) + state = DashboardState(output_dir=tmp_path, dashboard_spec=YAM_DASHBOARD_SPEC) + monkeypatch.setattr(dashboard_cli, "get_toolkit", get_toolkit) + monkeypatch.setattr( + dashboard_cli, + "build_planner", + lambda *args, **kwargs: SimpleNamespace(solve=solve), + ) + + error = dashboard_cli._run_dashboard_task( + args=args, + robot_spec=spec, + state=state, + claimed=ClaimedTask(number=1, request={}, output_dir=config.output_dir), + shared_runtime_kwargs={}, + unique_components=set(), + session_root=tmp_path / "session", + ) + + assert error is None + assert state.operator is None + assert closed == [True] + assert len(finalized) == 1 + assert finalized[0].environment_success is environment_success + assert finalized[0].agent_error is None diff --git a/tests/unit_tests/rpent/dashboard/test_state_contracts.py b/tests/unit_tests/rpent/dashboard/test_state_contracts.py index 425b66437..9e8c8f062 100644 --- a/tests/unit_tests/rpent/dashboard/test_state_contracts.py +++ b/tests/unit_tests/rpent/dashboard/test_state_contracts.py @@ -38,6 +38,12 @@ from rpent.session import EnvState from rpent.tools import Toolkit, ToolResult, tool + +@pytest.fixture(autouse=True) +def tool_template_output_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("rpent.utils.templates.get_output_dir", lambda: tmp_path) + + DASHBOARD_SPEC: DashboardSpec = { "task": { "command": "/rpent-task", @@ -81,6 +87,71 @@ def _claim_started_task( return claimed +@pytest.mark.parametrize("command", ["/done", "/success", "/failure", "/abort"]) +@pytest.mark.parametrize("web_operator", [False, True]) +def test_external_robot_operator_commands_keep_terminal_routing( + tmp_path, command, web_operator +): + from robots.yam.robot_spec import YAM_DASHBOARD_SPEC + + state = DashboardState(output_dir=tmp_path, dashboard_spec=YAM_DASHBOARD_SPEC) + if web_operator: + state.operator = MagicMock() + with pytest.raises(ValueError, match="Use the operator terminal"): + state.submit_input(command, operator_context={"generation": -1}) + if web_operator: + state.operator.reply.assert_not_called() + state.operator.close.assert_not_called() + else: + assert state.operator is None + + +def test_external_robot_continue_uses_toolkit_without_web_operator(tmp_path): + from robots.yam.robot_spec import YAM_DASHBOARD_SPEC + + state = DashboardState( + output_dir=tmp_path, + dashboard_spec={ + **DASHBOARD_SPEC, + "operator_commands": YAM_DASHBOARD_SPEC["operator_commands"], + }, + ) + state.shared_services_ready() + _claim_started_task(state) + state.set_planner_activity("idle", accepting_input=True) + toolkit = MagicMock(spec=Toolkit) + toolkit.exploration_continuation.return_value = "check episode and continue" + state.bind_toolkit(toolkit) + + assert state.submit_input("/continue", operator_context={"generation": -1}) == { + "command": "/continue" + } + + toolkit.exploration_continuation.assert_called_once_with(explicit=True) + assert state.claim_next_pending_message().text == "check episode and continue" + assert state.operator is None + + +@pytest.mark.parametrize("web_operator", [False, True]) +def test_task_commands_ignore_operator_reply_context(tmp_path, web_operator): + state = _ready_state(tmp_path) + _claim_started_task(state) + if web_operator: + state.operator = MagicMock() + state.operator.snapshot.return_value = None + + state.submit_input( + "/rpent-task place 2", + operator_context={"generation": -1, "request_id": "previous-request"}, + ) + + assert state.task_replacement_requested + assert state.snapshot()["pending_task"] == {"label": "place / seed 2"} + if web_operator: + state.operator.reply.assert_not_called() + state.operator.close.assert_not_called() + + def test_dashboard_task_commands_validate_and_latest_unclaimed_request_wins( tmp_path: Path, ) -> None: From d5198920b266e0c6a96ce6eab4f573a94e521ece Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 21:13:43 +0800 Subject: [PATCH 03/18] fix(dual-franka): terminal-lifecycle review slice --- robots/dual_franka/prompts/explore.py | 22 +- robots/dual_franka/toolkit.py | 115 +-- rpent/cli/attended.py | 200 +++++ rpent/cli/main.py | 175 +++-- rpent/cli/tui.py | 11 +- rpent/evaluation/result.py | 50 ++ rpent/planner/base.py | 6 +- rpent/planner/claude_code.py | 15 +- rpent/planner/codex.py | 79 +- rpent/session/input.py | 84 +++ rpent/session/transcript.py | 43 ++ rpent/tools/human_in_the_loop.py | 150 +++- rpent/tools/toolkit.py | 5 + .../robots/dual_franka/test_exploration.py | 702 ++++++++++++++++-- .../rpent/cli/test_main_contracts.py | 40 + .../rpent/cli/test_tui_contracts.py | 111 +++ .../rpent/evaluation/test_result.py | 28 + .../rpent/planner/test_claude_contracts.py | 93 +++ .../rpent/planner/test_codex_contracts.py | 100 ++- .../rpent/tools/test_human_in_the_loop.py | 96 ++- 20 files changed, 1864 insertions(+), 261 deletions(-) create mode 100644 rpent/cli/attended.py create mode 100644 rpent/session/input.py create mode 100644 rpent/session/transcript.py create mode 100644 tests/unit_tests/rpent/cli/test_tui_contracts.py diff --git a/robots/dual_franka/prompts/explore.py b/robots/dual_franka/prompts/explore.py index 4966fdbf8..d6e2a6f4b 100644 --- a/robots/dual_franka/prompts/explore.py +++ b/robots/dual_franka/prompts/explore.py @@ -29,6 +29,13 @@ state take precedence over using the remaining exploration budget.""" RULES = ( + "Acknowledge explicit operator feedback and explain the resulting plan change. " + "An operator may request practicing a later task stage while preserving earlier " + "progress. Request confirmation of that partial starting scene, not full restoration. " + "Do not repeat completed stages merely to satisfy the preset order. Still obtain " + "fresh reset confirmation before motion, inspect the scene and re-localize. " + "Record partial-start practice as such, not as an end-to-end successful run. " + "If feedback is ambiguous, ask a specific question instead of ignoring it.", "Before the first motion in every session, request_scene_reset(reason, " "expected_scene_state). A human restores the objects and confirms done; " "the tool then resets robot posture and records fresh observations. A new " @@ -36,17 +43,20 @@ "After reset, re-read the new state and re-localize. Never reuse a pixel, " "point or TCP target from an earlier attempt. Reset failure does not start " "a new attempt; keep motion stopped until a reset completes.", - "Read describe_dual_franka_setup for registered tools and cameras. Use move_delta, " - "rotate_delta, gripper tools, recover_joint_posture, back_project, optional " - "segment, and the named vla_right_grasp/vla_handoff/vla_left_place skills. " - "Do not assume LIBERO primitives or the older vla_grasp tool exist.", + "Read describe_dual_franka_setup for registered tools and cameras. Use only " + "the tools available for the current task; do not assume a fixed skill set.", "Ask request_operator_verdict after apparent success or failure. The human " "answers success, failure, continue or abort. Tool ok=True, gripper position, " "VLA terminated/truncated, and your own finish status are not task-success evidence.", "Any subsequent physical action invalidates the verdict. continue clears " - "the previous verdict. Obtain another judgment before finish.", + "the previous verdict. A returned continue is already authorization to " + "resume this attempt, not a request to wait for another continue. Read its " + "notes and fresh observation and continue without resetting. Do not ask " + "again about the same answered question unless new evidence introduces a " + "concrete blocker. Obtain another judgment before finish.", "For a failed attempt, archive evidence and update wip before requesting " - "a new scene reset. Change a named lever: order, staging, target or VLA prompt/chunk budget. " + "a new scene reset. Change one meaningful lever, such as perception inputs " + "or use of available tools, within the current task requirements. " "Use in-place recovery only when safe. Never force a restart after operator abort.", ) diff --git a/robots/dual_franka/toolkit.py b/robots/dual_franka/toolkit.py index c69a06c96..eb26226ed 100644 --- a/robots/dual_franka/toolkit.py +++ b/robots/dual_franka/toolkit.py @@ -83,11 +83,12 @@ def __init__( if attempts_per_session < 0: raise ValueError("attempts_per_session must be nonnegative") self._mode = mode + self._attended = mode == "exploration" or operator_input is not None self._attempt = 0 # first attempt begins only after scene confirmation - self._budget = attempts_per_session + self._budget = attempts_per_session if mode == "exploration" else 0 self._attempts_per_session = attempts_per_session self._session_attempt = 1 - self._scene_ready = mode != "exploration" + self._scene_ready = not self._attended self._operator_input = operator_input self._operator_verdict = None self._operator_notes = "" @@ -97,6 +98,8 @@ def __init__( self._events = [] self._direct_verdict_event = threading.Event() self._direct_verdict: str | None = None + self._direct_verdict_notes = "" + self._session_closed = threading.Event() super().__init__( runtime_kwargs=runtime_kwargs, dashboard_events=dashboard_events, @@ -109,32 +112,41 @@ def direct_verdict_requested(self) -> bool: """Whether operator control has sealed the active session.""" return self._direct_verdict_event.is_set() - def request_direct_verdict(self, verdict: str) -> bool: + def request_direct_verdict(self, verdict: str, notes: str = "") -> bool: """Seal this attempt immediately; cancel in-flight work at its next boundary.""" if verdict not in {"success", "failure", "abort"}: raise ValueError("verdict must be success, failure or abort") with self._scheduler.condition: + if self._session_closed.is_set(): + return False if self._direct_verdict_event.is_set(): - return self._direct_verdict == verdict - if self._mode != "exploration" or ( + return False + if not self._attended or ( verdict == "success" and (not self._scene_ready or self._operator_aborted) ): return False self._direct_verdict = verdict + self._direct_verdict_notes = notes self._direct_verdict_event.set() self._scheduler.cancel(close=True) return True def raise_if_cancelled(self) -> None: + if self._session_closed.is_set(): + raise ToolCancelled("exploration session closed") if self._direct_verdict_event.is_set(): raise ToolCancelled( "operator submitted a terminal verdict; stopping exploration" ) super().raise_if_cancelled() - def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: - if self._direct_verdict_event.is_set(): + def execute_tool( + self, + name: str, + input_dict: dict[str, Any], + ) -> ToolResult: + if self._direct_verdict_event.is_set() or self._session_closed.is_set(): return ToolResult( data={ "error": ( @@ -145,6 +157,12 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: ) return super().execute_tool(name, input_dict) + def close(self) -> None: + """Stop and join old tools before their operator input can be reused.""" + self._session_closed.set() + self.cancel_active_and_wait() + super().close() + def finalize_direct_verdict(self) -> dict[str, Any]: """Record fresh operator evidence after all active work has stopped.""" self.cancel_active_and_wait() @@ -160,21 +178,20 @@ def finalize_direct_verdict(self) -> dict[str, Any]: "operator_finished": True, "operator_aborted": True, "verdict_source": "interactive_command", + "operator_notes": self._direct_verdict_notes, } self._event("verdict", verdict="abort", source="interactive_command") self._event("finish", **result) return result - self.get_env_state( - command={"action": "observe_for_verdict"}, result={}, elapsed_s=0.0 - ) - self._validate_observation() + self._observe_current("observe_for_verdict") record = self.state.latest_record() - self._publish_step(record) self._scene_ready = True self._operator_verdict = self._direct_verdict self._operator_notes = ( f"Operator entered /{self._direct_verdict} in the interactive terminal." ) + if self._direct_verdict_notes: + self._operator_notes += " " + self._direct_verdict_notes self._verdict_step = record.step_idx self._event( "verdict", @@ -185,9 +202,10 @@ def finalize_direct_verdict(self) -> dict[str, Any]: result = { "status": self._direct_verdict, "operator_verdict": self._direct_verdict, - "operator_finished": True, + "operator_finished": False, "verdict_source": "interactive_command", "operator_aborted": False, + "operator_notes": self._direct_verdict_notes, } self._event("finish", **result) return result @@ -324,10 +342,11 @@ def _request_scene_reset( kind="reset", ) self._event("reset_response", response=response) - if response is None or response.strip().lower() != "done": - self._operator_aborted = ( - response is None or response.strip().lower() == "abort" - ) + response_parts = (response or "").strip().split(maxsplit=1) + confirmation = response_parts[0].lower() if response_parts else "" + notes = response_parts[1] if len(response_parts) > 1 else "" + if confirmation != "done": + self._operator_aborted = response is None or confirmation == "abort" return ToolResult( data={ "error": "scene reset not confirmed", @@ -350,6 +369,7 @@ def _request_scene_reset( "ok": True, "robot_reset": result, "scene_reset_confirmed": True, + "operator_notes": notes, "notice": "Scene restored by operator; robot posture reset. Re-localize from the new images.", } ) @@ -377,12 +397,8 @@ def _request_operator_verdict( data={"error": "verdict refused; no active confirmed attempt"} ) # Save the evidence being judged using the existing camera/state logger. - self.get_env_state( - command={"action": "observe_for_verdict"}, result={}, elapsed_s=0.0 - ) - self._validate_observation() + self._observe_current("observe_for_verdict") record = self.state.latest_record() - self._publish_step(record) response = self._ask_operator( f"{question}\nAttempt {self._attempt}, observation step {record.step_idx}. " "Reply success, failure, continue, or abort; optional notes may follow.", @@ -399,7 +415,19 @@ def _request_operator_verdict( self._operator_verdict = verdict self._operator_notes = notes self._verdict_step = record.step_idx - elif verdict != "continue": + elif verdict == "continue": + observation = self._view_env_state() + observation.data.update( + ok=True, + status=verdict, + operator_notes=notes, + attempt=self._attempt, + evidence_step=record.step_idx, + observation_step=self.state.latest_step, + operator_aborted=False, + ) + return observation + else: return ToolResult( data={"error": "invalid operator verdict", "evidence": event} ) @@ -440,8 +468,8 @@ def _guarded_finish(self, inner, **kwargs) -> ToolResult: self._event("finish", **result.to_dict()) return result - def get_env_state(self, *, command, result, elapsed_s): - if self._mode != "exploration": + def get_env_state(self, *, command, result, elapsed_s) -> ToolResult: + if not self._attended: return super().get_env_state( command=command, result=result, elapsed_s=elapsed_s ) @@ -464,8 +492,11 @@ def get_env_state(self, *, command, result, elapsed_s): "operator_verdict": self._operator_verdict, "attempt_start_step": self._attempt_start_step, } - self.state.save("exploration.json", status) - output.data["exploration"] = status + lifecycle = ( + "exploration" if self._mode == "exploration" else "operator_lifecycle" + ) + self.state.save(f"{lifecycle}.json", status) + output.data[lifecycle] = status if result.get("error"): output.data["error"] = result["error"] return output @@ -500,7 +531,7 @@ def _validate_observation(self) -> None: def solved(self) -> bool: if self.flash_options is not None: return self._flash_solved - if self._mode != "exploration": + if not self._attended: return self._operator_verdict == "success" return ( self._scene_ready @@ -587,16 +618,16 @@ def _register_tools(self) -> None: ), "request_scene_reset": self._request_scene_reset, "request_operator_verdict": self._request_operator_verdict - if self._mode == "exploration" + if self._attended else self._evaluation_verdict, } ) for definition in self.declared_tools(): name = definition.name - if name in _EXPLORATION_ONLY_TOOLS and self._mode != "exploration": + if name in _EXPLORATION_ONLY_TOOLS and not self._attended: continue handler = state_handlers.get(name) or getattr(self._primitives, name) - if self._mode == "exploration": + if self._attended: if name in _MOTION_TOOLS: handler = partial(self._guard_motion, handler) elif name in {"back_project", "segment"}: @@ -605,11 +636,7 @@ def _register_tools(self) -> None: handler = partial(self._describe_exploration_setup, handler) self.add_tool(definition.with_handler(handler)) finish = self._tools["finish"] - guard = ( - self._guarded_finish - if self._mode == "exploration" - else self._evaluation_finish - ) + guard = self._guarded_finish if self._attended else self._evaluation_finish self.add_tool(finish.with_handler(partial(guard, finish)), replace=True) def _read_operator_line(self, prompt: str) -> str | None: @@ -650,15 +677,15 @@ def _evaluation_verdict( } ) if verdict == "continue": - return ToolResult( - data={ - "ok": True, - "status": "continue", - "operator_notes": notes, - "attempt": self._attempt, - "notice": "Operator requested more action; do not finish yet.", - } + observation = self._view_env_state() + observation.data.update( + ok=True, + status="continue", + operator_notes=notes, + attempt=self._attempt, + notice="Operator requested more action; do not finish yet.", ) + return observation self._operator_verdict = verdict self._operator_notes = notes return ToolResult( diff --git a/rpent/cli/attended.py b/rpent/cli/attended.py new file mode 100644 index 000000000..a03519235 --- /dev/null +++ b/rpent/cli/attended.py @@ -0,0 +1,200 @@ +# Copyright 2026 The RPent Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI orchestration for one operator-attended attempt.""" + +import json +import shutil +import time +from pathlib import Path + +from rpent.cli.tui import next_user_line +from rpent.evaluation.result import record_operator_outcome +from rpent.session.transcript import _serialize_messages +from rpent.utils.logging import get_logger + +logger = get_logger("agent") + + +def _finalize_operator_verdict( + result, *, toolkit, memory, recipe_tag, output_dir, state_output_dir, session_number +): + """Archive the interrupted attempt and prepare its next-session handoff.""" + finish = toolkit.finalize_direct_verdict() + error = result.error + if error: + logger.info("Planner stopped after operator verdict: %s", error) + result.stats["planner_error_at_operator_verdict"] = error + verdict = finish["operator_verdict"] + handoff = "" + if verdict in {"success", "failure"}: + evidence = record_operator_outcome( + memory_root=memory.root, + recipe_tag=recipe_tag, + run_name=Path(output_dir).name, + session_number=session_number, + state_output_dir=state_output_dir, + verdict=finish, + messages=_serialize_messages(result.messages), + planner_error=error, + ) + handoff = ( + f"\nThe operator marked the preceding attempt as {verdict}. " + f"Before requesting any reset or motion, read {evidence}, " + "inspect its evidence and write a grounded outcome summary " + "to your memory inbox wip notes. Distinguish observations " + "from hypotheses; never label a failed attempt as successful. " + "Then request_scene_reset and wait for a new operator " + "confirmation before attempting the task again.\n" + ) + if verdict == "failure": + error = None + return finish, error, handoff + + +def _publish_success_recipe( + memory, *, recipe_tag, output_dir, state_output_dir, auto_merge +): + """Preserve successful-session artifacts before another session overwrites them.""" + for name in (f"{recipe_tag}.json", f"{recipe_tag}_recipe.jsonl"): + shutil.copy2(Path(output_dir) / name, Path(state_output_dir) / name) + if auto_merge: + memory.merge_memory( + cell_tag=recipe_tag, run_state_dir=state_output_dir, solved=True + ) + + +def _solve_attended( + planner, *, operator_input, state_output_dir, keep_mcp_alive=False, **kwargs +): + """Own the transport for the whole attended attempt, including human waits.""" + server = None + if keep_mcp_alive: + from rpent.planner.utils.http_mcp_server import HttpMcpServer + + server = HttpMcpServer(kwargs["toolkit"]) + kwargs["mcp_server"] = server + try: + if server is not None: + server.start() + return _solve_attended_turns( + planner, + operator_input=operator_input, + state_output_dir=state_output_dir, + **kwargs, + ) + finally: + if server is not None: + # Release pending tool handlers before asking HTTP to shut down. + kwargs["toolkit"].cancel_active_and_wait() + server.stop() + + +def _solve_attended_turns(planner, *, operator_input, state_output_dir, **kwargs): + """Keep an unjudged attempt alive across planner replies and human pauses.""" + toolkit = kwargs["toolkit"] + original_message = kwargs["user_message"].removesuffix( + operator_input.feedback_prompt() + ) + messages = [] + usage = {} + turn_number = 0 + while True: + continue_revision = operator_input.continue_revision + result = planner.solve(**kwargs) + turn_number += 1 + messages.extend(result.messages) + for key in ( + "total_input_tokens", + "total_output_tokens", + "total_cached_input_tokens", + "tool_calls", + ): + usage[key] = usage.get(key, 0) + result.stats.get(key, 0) + # Preserve every reply before the planner reuses its output filenames. + archive = Path(state_output_dir) / f"dialogue_{turn_number:03d}.json" + archive.parent.mkdir(parents=True, exist_ok=True) + archive.write_text( + json.dumps( + { + "messages": _serialize_messages(result.messages), + "stats": result.stats, + "finish": result.finish_result, + "error": result.error, + }, + indent=2, + default=str, + ) + ) + if result.error or result.finish_result or toolkit.direct_verdict_requested: + break + if operator_input.closed: + toolkit.request_direct_verdict("abort", "Interactive input closed") + break + resumed_operator_request = operator_input.pending_kind is not None + if resumed_operator_request: + logger.info( + "Waiting for the current operator request; keeping this attempt alive." + ) + while ( + operator_input.pending_kind is not None + and not toolkit.direct_verdict_requested + and not operator_input.closed + ): + time.sleep(0.1) + toolkit.wait_active() + if toolkit.direct_verdict_requested: + break + if operator_input.closed: + toolkit.request_direct_verdict("abort", "Interactive input closed") + break + resumed_operator_request |= ( + operator_input.continue_revision != continue_revision + ) + if resumed_operator_request: + reply = ( + "The pending operator request has returned or the operator already answered continue. " + "Read its recorded result and continue this attempt without waiting for the same reply again." + ) + else: + logger.info( + "Waiting for feedback or /continue; /success, /failure and /abort remain available." + ) + reply = next_user_line(kwargs["input_queue"]) + if toolkit.direct_verdict_requested: + break + if reply is None: + toolkit.request_direct_verdict("abort", "Interactive input closed") + break + if not resumed_operator_request: + observation = toolkit.execute_tool("view_env_state", {}).data + if observation.get("error"): + result.error = ( + f"Observation refresh failed after feedback: {observation['error']}" + ) + break + messages.append({"role": "user", "content": reply}) + kwargs["user_message"] = ( + original_message + + "\nContinue the SAME attempt, not a new exploration session. Robot state and " + "scene confirmation are retained; do not reset merely because the conversation resumed. " + f"Current attempt state and operator results are recorded under {state_output_dir}. " + "If the operator changed the scene, obtain fresh observations and localization. " + "Previous conversation:\n" + + json.dumps(_serialize_messages(messages), ensure_ascii=False, default=str) + + operator_input.feedback_prompt() + ) + result.messages = messages + result.stats.update(usage) + return result diff --git a/rpent/cli/main.py b/rpent/cli/main.py index cb53ec75a..9913c35f7 100644 --- a/rpent/cli/main.py +++ b/rpent/cli/main.py @@ -44,6 +44,11 @@ from collections.abc import Callable from pathlib import Path +from rpent.cli.attended import ( + _finalize_operator_verdict, + _publish_success_recipe, + _solve_attended, +) from rpent.cli.tui import ( start_first_prompt_resolver, start_interactive_reader, @@ -54,47 +59,21 @@ ) from rpent.evaluation import RunFinalizationContext from rpent.memory import MemoryManager -from rpent.planner.base import REASONING_EFFORTS, build_planner +from rpent.planner.base import ( + CODEX_REASONING_EFFORTS, + REASONING_EFFORTS, + build_planner, +) from rpent.planner.check import BASE_URL_ENV_BY_PLANNER from rpent.robots import enumerate_robots, get_robot_spec, get_toolkit +from rpent.session.input import SessionInputQueue +from rpent.session.transcript import _serialize_messages +from rpent.session.transcript import _strip_images as _strip_images from rpent.utils.logging import get_logger, init_output_dir logger = get_logger("agent") -# --------------------------------------------------------------------------- -# API agent transcript serialization -# --------------------------------------------------------------------------- - - -def _strip_images(value): - """Return a copy of ``value`` with inline image payloads omitted. - - SDK objects are left untouched; ``json.dump(..., default=str)`` handles - them at write time. Only the bulky base64 image blocks are replaced. - """ - if isinstance(value, list): - return [_strip_images(v) for v in value] - if isinstance(value, dict): - if value.get("type") == "image": - return {"type": "image", "source": {"_omitted_for_transcript": True}} - if value.get("type") == "image_url": - return {"type": "image_url", "image_url": {"_omitted_for_transcript": True}} - return {k: _strip_images(v) for k, v in value.items()} - return value - - -def _serialize_messages(messages: list[dict]) -> list[dict]: - """Strip inline image payloads from messages before writing the transcript.""" - return [ - { - **{k: v for k, v in m.items() if k != "content"}, - "content": _strip_images(m.get("content")), - } - for m in messages - ] - - # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- @@ -154,11 +133,12 @@ def _build_argparser() -> argparse.ArgumentParser: ap.add_argument("--max-tokens", type=int, default=8192) ap.add_argument( "--reasoning-effort", - choices=REASONING_EFFORTS, + choices=CODEX_REASONING_EFFORTS, default="none", help="Planner reasoning effort for api, claude_code, and " "codex. Higher effort may improve task success rate " - "but increases runtime. Defaults to none.", + "but increases runtime. Ultra is Codex-only and requires a supporting " + "model and SDK. Defaults to none.", ) ap.add_argument( "--no-images", @@ -175,7 +155,7 @@ def _build_argparser() -> argparse.ArgumentParser: help="Wall-clock cap for api/claude_code/codex planner runs. " "Terminal interactive API/Claude sessions are exempt. " "Defaults to CODEX_TIMEOUT_S (codex only), " - "CELL_TIMEOUT_S, or 1200.", + "CELL_TIMEOUT_S, or 1200. For codex, 0 disables this deadline.", ) ap.add_argument( "--claude-code-max-budget-usd", @@ -356,6 +336,10 @@ def main() -> int: ) args = parser.parse_args() args.robot_name = early.robot_name + if args.planner != "codex" and args.reasoning_effort not in REASONING_EFFORTS: + parser.error( + f"--reasoning-effort {args.reasoning_effort} requires --planner codex" + ) human_interactive_exploration = ( args.explore and robot_spec.supports_human_interactive_exploration ) @@ -476,9 +460,9 @@ def main() -> int: if human_interactive_exploration: from rpent.tools.human_in_the_loop import HumanInTheLoopInput - # The native CLI reads between runs, leaving the TTY available to tools. operator_input = HumanInTheLoopInput( - interactive=args.interactive and not native_cli + interactive=args.interactive and not native_cli, + feedback_path=Path(output_dir) / "operator_feedback.jsonl", ) input_queue: "queue.Queue[str | None] | None" = None await_first_prompt: "Callable[[], str | None] | None" = None @@ -538,11 +522,15 @@ def main() -> int: environment_success: bool | None = None memory_manager: MemoryManager | None = None direct_operator_success = False + verdict_handoff = "" try: if first_user_msg is not None: dashboard_events.emit(RunStartedEvent()) session_msg = first_user_msg for session_number in range(1, sessions + 1): + direct_operator_success = False + continue_after_verdict = False + solved = False if session_msg is None: break if session_number > 1: @@ -556,6 +544,9 @@ def main() -> int: session_number=session_number, session_max=sessions, ) + session_msg += verdict_handoff + if operator_input is not None and args.interactive: + session_msg += operator_input.feedback_prompt() state_output_dir = output_dir if getattr(args, "explore", False): state_output_dir = ( @@ -586,51 +577,90 @@ def main() -> int: config=run_config, ) memory_manager = toolkit.memory + session_input = input_queue if operator_input is not None and input_queue is not None: - - def accept_verdict(verdict: str, active_toolkit=toolkit) -> bool: - if not active_toolkit.request_direct_verdict(verdict): + session_input = SessionInputQueue(input_queue) + + def accept_verdict( + verdict: str, + notes: str, + active_toolkit=toolkit, + active_input=session_input, + ) -> bool: + if not active_toolkit.request_direct_verdict(verdict, notes): return False logger.info( - "/%s accepted: stopping actions, then recording the result and exiting.", + "/%s accepted: stopping this attempt and recording its result.", verdict, ) - # EOF is a planner control signal, never a model message. - input_queue.put(None) + active_input.cancel() return True operator_input.bind_verdict(accept_verdict) + pending_operator_request = None try: - result = planner.solve( + solve = planner.solve + if human_interactive_exploration and input_queue is not None: + from functools import partial + + solve = partial( + _solve_attended, + planner, + keep_mcp_alive=args.planner == "codex", + operator_input=operator_input, + state_output_dir=state_output_dir, + ) + result = solve( system_prompt=system_prompt, user_message=session_msg, toolkit=toolkit, max_turns=args.max_turns, - input_queue=input_queue, + input_queue=session_input, ) finish_result = result.finish_result messages += result.messages stats = result.stats agent_error = result.error - if getattr(toolkit, "direct_verdict_requested", False): - finish_result = toolkit.finalize_direct_verdict() - direct_operator_success = finish_result["status"] == "success" - if agent_error: - logger.info( - "Planner stopped after operator verdict: %s", agent_error + if operator_input is not None and args.interactive: + pending_operator_request = operator_input.pending_kind + if human_interactive_exploration and toolkit.direct_verdict_requested: + finish_result, agent_error, verdict_handoff = ( + _finalize_operator_verdict( + result, + toolkit=toolkit, + memory=memory_manager, + recipe_tag=recipe_tag, + output_dir=output_dir, + state_output_dir=state_output_dir, + session_number=session_number, ) - stats["planner_error_at_operator_verdict"] = agent_error - solved_fn = getattr(toolkit, "solved", None) - if getattr(robot_spec, "supports_exploration", False) and callable( - solved_fn - ): - solved = bool(solved_fn()) - write_recipe = getattr(toolkit, "write_recipe", None) - if solved and callable(write_recipe): - recipe_path = write_recipe(recipe_tag) or recipe_path + ) + direct_operator_success = finish_result["status"] == "success" + continue_after_verdict = ( + finish_result["operator_verdict"] in {"success", "failure"} + and not agent_error + ) + if robot_spec.supports_exploration: + solved = bool(toolkit.solved()) + if solved: + recipe_path = toolkit.write_recipe(recipe_tag) or recipe_path + if ( + human_interactive_exploration + and direct_operator_success + and not agent_error + ): + _publish_success_recipe( + memory_manager, + recipe_tag=recipe_tag, + output_dir=output_dir, + state_output_dir=state_output_dir, + auto_merge=args.auto_merge_memory, + ) finally: - if operator_input is not None and input_queue is not None: + if operator_input is not None and args.interactive: operator_input.bind_verdict(None) + if session_input is not None: + session_input.cancel() try: if robot_spec.finalize_run is not None: solved_fn = getattr(toolkit, "solved", None) @@ -640,8 +670,23 @@ def accept_verdict(verdict: str, active_toolkit=toolkit) -> bool: solved = bool(environment_success) finally: toolkit.close() + if operator_input is not None and args.interactive: + operator_input.cancel_pending() + if ( + human_interactive_exploration + and not continue_after_verdict + and not (finish_result or {}).get("_finish") + and not (finish_result or {}).get("operator_finished") + ): + agent_error = agent_error or ( + f"Planner returned without an attempt verdict; pending operator request: " + f"{pending_operator_request or 'none'}. Old tools were cancelled. " + "Stopping exploration instead of starting another empty session." + ) + logger.error("%s", agent_error) + break if ( - solved + (solved and not continue_after_verdict) or (finish_result or {}).get("operator_aborted") or (finish_result or {}).get("operator_finished") ): @@ -727,7 +772,7 @@ def accept_verdict(verdict: str, active_toolkit=toolkit) -> bool: # Publish exploration artifacts into the corpus after the session loop. if ( getattr(args, "explore", False) - and (getattr(args, "auto_merge_memory", False) or direct_operator_success) + and getattr(args, "auto_merge_memory", False) and not agent_error and memory_manager is not None and (not human_interactive_exploration or solved) diff --git a/rpent/cli/tui.py b/rpent/cli/tui.py index 747f5dfa4..5a59b3642 100644 --- a/rpent/cli/tui.py +++ b/rpent/cli/tui.py @@ -19,7 +19,6 @@ import atexit import contextlib import logging -import queue import sys import threading from collections.abc import Callable @@ -31,6 +30,8 @@ from prompt_toolkit.patch_stdout import patch_stdout from prompt_toolkit.styles import Style +from rpent.session.input import InputQueue + #: Interactive-mode command tokens (case-insensitive). This module is the single #: source of truth; ``api_loop`` imports ``QUIT_TOKENS`` for its steering checks. QUIT_TOKENS = frozenset({"/quit", "/exit", "/q"}) @@ -149,7 +150,7 @@ def _restore() -> None: def start_interactive_reader( - input_queue: "queue.Queue[str | None]", + input_queue: InputQueue, *, first_prompt_default: str | None = None, line_handler: Callable[[str], bool] | None = None, @@ -203,7 +204,7 @@ def _read() -> None: return thread -def next_user_line(input_queue: "queue.Queue[str | None]") -> str | None: +def next_user_line(input_queue: InputQueue) -> str | None: """Block for the next actionable user line from an interactive input queue. Returns the trimmed line, or ``None`` when the session should end (the queue @@ -223,7 +224,7 @@ def next_user_line(input_queue: "queue.Queue[str | None]") -> str | None: def initial_user_message( - input_queue: "queue.Queue[str | None]", + input_queue: InputQueue, ) -> str | None: """Block for the first user turn of an interactive session. @@ -243,7 +244,7 @@ def initial_user_message( def start_first_prompt_resolver( - input_queue: "queue.Queue[str | None]", + input_queue: InputQueue, ) -> Callable[[], str | None]: """Resolve the opening user turn on a background thread. diff --git a/rpent/evaluation/result.py b/rpent/evaluation/result.py index 897798f83..7d0c88191 100644 --- a/rpent/evaluation/result.py +++ b/rpent/evaluation/result.py @@ -23,6 +23,8 @@ from pathlib import Path from typing import Any +from rpent.utils.logging import get_logger + @dataclass(frozen=True) class RunFinalizationContext: @@ -46,6 +48,54 @@ class RunFinalizationContext: RunFinalizer = Callable[[RunFinalizationContext], Path | str | None] +def record_operator_outcome( + *, + memory_root: Path, + recipe_tag: str, + run_name: str, + session_number: int, + state_output_dir: str | Path, + verdict: dict[str, Any], + messages: list[dict[str, Any]], + planner_error: str | None, +) -> Path: + """Archive an operator outcome without overwriting a previous attempt. + + Args: + memory_root: Root of the run's memory store. + recipe_tag: Cell whose inbox receives the evidence. + run_name: Run identifier used to distinguish evidence files. + session_number: Attempt number within the run. + state_output_dir: Directory containing the attempt's observations. + verdict: Operator-confirmed result and notes. + messages: Serializable planner transcript. + planner_error: Error recorded before operator finalization, if any. + + Returns: + Path to the archived evidence JSON. + """ + outcome = verdict["operator_verdict"] + path = memory_root / "_internal" / "inbox" / recipe_tag / "wip" + path /= f"{run_name}-session-{session_number:03d}-{outcome}.json" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x") as file: + json.dump( + { + "verdict": verdict, + "session": session_number, + "evidence_dir": str(state_output_dir), + "messages": messages, + "planner_error": planner_error, + "note": "Operator-confirmed outcome; explanations still require evidence review.", + }, + file, + indent=2, + default=str, + ) + get_logger("evaluation").info("Attempt evidence saved: %s", path) + return path + + def write_json_atomic( destination: Path | str, record: Mapping[str, Any], diff --git a/rpent/planner/base.py b/rpent/planner/base.py index 4e7930a34..c55d355ec 100644 --- a/rpent/planner/base.py +++ b/rpent/planner/base.py @@ -18,7 +18,6 @@ import asyncio import os -import queue from abc import ABC, abstractmethod from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor @@ -37,10 +36,13 @@ if TYPE_CHECKING: from pydantic_ai.models import Model + from rpent.session.input import InputQueue + #: MCP namespace prefix for RPent tools (``mcp____``). #: Toolkits expose plain tool names; planners add/strip this prefix. MCP_TOOL_PREFIX = "mcp__rpent__" REASONING_EFFORTS = ("none", "low", "medium", "high", "xhigh") +CODEX_REASONING_EFFORTS = (*REASONING_EFFORTS, "ultra") def resolve_model(planner: str, model: str | None) -> str | None: @@ -135,7 +137,7 @@ def solve( user_message: str, toolkit: Toolkit, max_turns: int, - input_queue: queue.Queue[str | None] | None = None, + input_queue: InputQueue | None = None, dashboard_interaction: DashboardInteractionPort | None = None, ) -> PlannerResult: """Run the multi-turn agent loop until completion or budget. diff --git a/rpent/planner/claude_code.py b/rpent/planner/claude_code.py index d93120377..8b2f9f71c 100644 --- a/rpent/planner/claude_code.py +++ b/rpent/planner/claude_code.py @@ -27,7 +27,6 @@ import contextlib import dataclasses import json -import queue import tempfile import time from collections.abc import Callable @@ -53,6 +52,7 @@ strip_mcp_prefix, ) from rpent.planner.utils.http_mcp_server import mcp_result +from rpent.session.input import InputQueue, SessionInputQueue from rpent.tools import Toolkit, ToolResult from rpent.utils.config import get_repo_root from rpent.utils.logging import get_logger, init_output_dir @@ -105,7 +105,7 @@ def solve( user_message: str, toolkit: Toolkit, max_turns: int, - input_queue: queue.Queue[str | None] | None = None, + input_queue: InputQueue | None = None, dashboard_interaction: DashboardInteractionPort | None = None, ) -> PlannerResult: """Run a Claude Agent SDK session for the given prompt.""" @@ -134,7 +134,7 @@ async def _solve_async( initial_user_text: str, toolkit: Toolkit, max_turns: int, - input_queue: queue.Queue[str | None] | None = None, + input_queue: InputQueue | None = None, dashboard_interaction: DashboardInteractionPort | None = None, ) -> PlannerResult: import claude_agent_sdk @@ -501,9 +501,9 @@ async def _consume(self, adapter: Any) -> None: class _TerminalSessionAdapter: """Preserve the terminal TUI's interrupt-then-query steering policy.""" - def __init__(self, *, toolkit: Toolkit, input_queue: Any, emit_user) -> None: + def __init__(self, *, toolkit: Toolkit, input_queue: InputQueue, emit_user) -> None: self._toolkit = toolkit - self._input_queue = input_queue + self._input_queue = SessionInputQueue(input_queue) self._emit_user = emit_user async def initial_query_succeeded(self, driver: _ClaudeSessionDriver) -> None: @@ -512,6 +512,8 @@ async def initial_query_succeeded(self, driver: _ClaudeSessionDriver) -> None: async def run(self, driver: _ClaudeSessionDriver) -> None: while True: nxt = await asyncio.to_thread(next_user_line, self._input_queue) + if self._input_queue.cancelled: + return await cancel_and_wait(self._toolkit.cancel_active_and_wait) await driver.interrupt() if nxt is None: @@ -528,8 +530,7 @@ async def on_message( return None async def close(self) -> None: - # Unblock next_user_line() if the consumer (for example finish) won. - self._input_queue.put(None) + self._input_queue.cancel() class _ClaudeDashboardAdapter: diff --git a/rpent/planner/codex.py b/rpent/planner/codex.py index f3eca67c7..def6316c6 100644 --- a/rpent/planner/codex.py +++ b/rpent/planner/codex.py @@ -29,7 +29,6 @@ import contextlib import json import os -import queue import tempfile import threading import time @@ -49,13 +48,14 @@ from rpent.dashboard.interaction import DashboardInteractionPort, DashboardMessage from rpent.dashboard.planner_control import DashboardPlannerControl from rpent.planner.base import ( - REASONING_EFFORTS, + CODEX_REASONING_EFFORTS, Planner, PlannerResult, cancel_and_wait, strip_mcp_prefix, ) from rpent.planner.utils.http_mcp_server import HttpMcpServer +from rpent.session.input import InputQueue, SessionInputQueue from rpent.tools.toolkit import Toolkit from rpent.utils.config import get_repo_root from rpent.utils.logging import get_logger @@ -105,7 +105,9 @@ def __init__( """Initialize the Codex SDK backend.""" self._output_dir = str(output_dir) self._repo_root = str(repo_root) if repo_root else str(get_repo_root()) - self._timeout_s = timeout_s + if timeout_s < 0: + raise ValueError("timeout_s must be nonnegative (0 disables the deadline)") + self._timeout_s = None if timeout_s == 0 else timeout_s self._extra_dirs = extra_dirs or [] self._output_path = Path(output_path) if output_path else None self._model = model or os.environ.get("CODEX_MODEL", None) @@ -113,7 +115,7 @@ def __init__( self._api_key = os.environ.get("CODEX_API_KEY", None) self._service_tier = os.environ.get("CODEX_SERVICE_TIER", None) self._dashboard_events = dashboard_events - if reasoning_effort not in REASONING_EFFORTS: + if reasoning_effort not in CODEX_REASONING_EFFORTS: raise ValueError(f"unsupported reasoning effort: {reasoning_effort}") self._thread_options = { "approval_mode": openai_codex.ApprovalMode.deny_all, @@ -135,8 +137,9 @@ def solve( user_message: str, toolkit: Toolkit, max_turns: int, - input_queue: queue.Queue[str | None] | None = None, + input_queue: InputQueue | None = None, dashboard_interaction: DashboardInteractionPort | None = None, + mcp_server: HttpMcpServer | None = None, ) -> PlannerResult: """Run one or more Codex SDK turns for the given prompt.""" if input_queue is not None and dashboard_interaction is not None: @@ -145,6 +148,8 @@ def solve( ) prompt = f"{system_prompt}\n\n{user_message}" if system_prompt else user_message if dashboard_interaction is not None: + if mcp_server is not None: + raise ValueError("dashboard owns its MCP server") return asyncio.run( self._solve_dashboard( prompt=prompt, @@ -163,7 +168,9 @@ def solve( # Start the in-thread MCP HTTP server so Codex can reach the # shared toolkit without spawning a subprocess. - mcp_server = HttpMcpServer(toolkit) + owns_mcp_server = mcp_server is None + if owns_mcp_server: + mcp_server = HttpMcpServer(toolkit) mcp_url = mcp_server.start() logger.info("mcp http endpoint: %s", mcp_url) @@ -171,9 +178,9 @@ def solve( logger.info("prompt: %d chars", len(prompt)) logger.info("output_dir: %s", self._output_dir) logger.info( - "invoking Codex SDK model %s (timeout=%ds)", + "invoking Codex SDK model %s (timeout=%s)", model_desc, - self._timeout_s, + "unlimited" if self._timeout_s is None else f"{self._timeout_s}s", ) started = time.time() @@ -225,13 +232,14 @@ def solve( _write_jsonl(raw_f, {"type": "error", "message": error}) logger.info(rendered.rstrip()) finally: - try: - toolkit.cancel_active_and_wait() - except Exception as exc: - logger.exception("Codex toolkit cleanup failed") - error = error or f"Toolkit cleanup failed: {exc}" - finally: - mcp_server.stop() + if owns_mcp_server: + try: + toolkit.cancel_active_and_wait() + except Exception as exc: + logger.exception("Codex toolkit cleanup failed") + error = error or f"Toolkit cleanup failed: {exc}" + finally: + mcp_server.stop() elapsed = time.time() - started text = state.get("text", "") or output_path.read_text(errors="replace") @@ -284,7 +292,7 @@ def _run_session( state: dict[str, Any], mcp_url: str, toolkit: Toolkit, - input_queue: "queue.Queue[str | None] | None" = None, + input_queue: InputQueue | None = None, ) -> None: try: chunks: list[str] = [] @@ -302,21 +310,24 @@ def _run_session( turn = thread.turn(prompt, **self._turn_options) state["turn"] = turn - stop_steer: threading.Event | None = None + steering_input: SessionInputQueue | None = None + steer_thread: threading.Thread | None = None if input_queue is not None: - stop_steer = threading.Event() + steering_input = SessionInputQueue(input_queue) def _steer() -> None: while True: - nxt = next_user_line(input_queue) - if stop_steer.is_set(): - return + nxt = next_user_line(steering_input) if nxt is None: - toolkit.cancel_active_and_wait() - try: - turn.interrupt() - except Exception: - pass + if not steering_input.cancelled: + toolkit.cancel_active_and_wait() + try: + turn.interrupt() + except Exception: + pass + return + if steering_input.cancelled: + input_queue.put(nxt) return rendered = f"\n[user] {nxt}\n" with write_lock: @@ -326,6 +337,9 @@ def _steer() -> None: logger.info(rendered.strip()) try: turn.steer(nxt) + logger.info( + "[feedback] delivered to active Codex turn" + ) except Exception as e: rendered = f"\n[codex-planner] steer failed: {e}\n" with write_lock: @@ -335,11 +349,12 @@ def _steer() -> None: logger.info(rendered.strip()) return - threading.Thread( + steer_thread = threading.Thread( target=_steer, name="codex-steer", daemon=True, - ).start() + ) + steer_thread.start() limit_reached = False try: @@ -365,10 +380,10 @@ def _steer() -> None: raise # The terminal notification may still be queued. finally: - if stop_steer is not None: - stop_steer.set() - if input_queue is not None: - input_queue.put(None) + if steering_input is not None: + steering_input.cancel() + if steer_thread is not None: + steer_thread.join(timeout=2) state["text"] = "".join(chunks) if recorder.final_response is not None: diff --git a/rpent/session/input.py b/rpent/session/input.py new file mode 100644 index 000000000..fb8b14eeb --- /dev/null +++ b/rpent/session/input.py @@ -0,0 +1,84 @@ +# Copyright 2026 The RPent Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Input queue contracts and cancellation scopes for attended sessions.""" + +from __future__ import annotations + +import queue +import threading +import time +from typing import Protocol + + +class InputQueue(Protocol): + """Input operations shared by readers and planner consumers.""" + + def get(self, block: bool = True, timeout: float | None = None) -> str | None: + """Read a submitted line or the source's EOF signal.""" + ... + + def put(self, item: str | None) -> None: + """Forward a line or the source's EOF signal.""" + ... + + +class SessionInputQueue: + """Cancel one input consumer without putting EOF in the shared queue. + + A scope may wrap another scope, allowing a planner reply to end without + cancelling its attended attempt or consuming the next attempt's input. + """ + + def __init__(self, source: InputQueue) -> None: + """Wrap a source queue without taking ownership of its lifetime.""" + self._source = source + self._cancelled = threading.Event() + self._lock = threading.Lock() + + @property + def cancelled(self) -> bool: + """Return whether this scope has been cancelled.""" + return self._cancelled.is_set() + + def cancel(self) -> None: + """Wake this scope's readers without closing or changing its source.""" + with self._lock: + self._cancelled.set() + + def get(self, block: bool = True, timeout: float | None = None) -> str | None: + """Read input, returning None when this scope or its source closes.""" + if block and timeout is not None and timeout < 0: + raise ValueError("timeout must be a non-negative number") + deadline = time.monotonic() + timeout if timeout is not None else None + while True: + with self._lock: + if self.cancelled: + return None + try: + return self._source.get(block=False) + except queue.Empty: + if not block: + raise + wait_time = 0.1 + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise queue.Empty + wait_time = min(wait_time, remaining) + self._cancelled.wait(wait_time) + + def put(self, item: str | None) -> None: + """Forward real input to the source, including returned unread lines.""" + self._source.put(item) diff --git a/rpent/session/transcript.py b/rpent/session/transcript.py new file mode 100644 index 000000000..0c3f7f01f --- /dev/null +++ b/rpent/session/transcript.py @@ -0,0 +1,43 @@ +# Copyright 2026 The RPent Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Transcript serialization without inline image payloads.""" + + +def _strip_images(value): + """Return a copy of ``value`` with inline image payloads omitted. + + SDK objects are left untouched; ``json.dump(..., default=str)`` handles + them at write time. Only the bulky base64 image blocks are replaced. + """ + if isinstance(value, list): + return [_strip_images(v) for v in value] + if isinstance(value, dict): + if value.get("type") == "image": + return {"type": "image", "source": {"_omitted_for_transcript": True}} + if value.get("type") == "image_url": + return {"type": "image_url", "image_url": {"_omitted_for_transcript": True}} + return {k: _strip_images(v) for k, v in value.items()} + return value + + +def _serialize_messages(messages: list[dict]) -> list[dict]: + """Strip inline image payloads from messages before writing the transcript.""" + return [ + { + **{k: v for k, v in m.items() if k != "content"}, + "content": _strip_images(m.get("content")), + } + for m in messages + ] diff --git a/rpent/tools/human_in_the_loop.py b/rpent/tools/human_in_the_loop.py index 6b0126fa0..5e7a04f47 100644 --- a/rpent/tools/human_in_the_loop.py +++ b/rpent/tools/human_in_the_loop.py @@ -16,18 +16,60 @@ from __future__ import annotations +import json import queue import select import sys import threading import uuid from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path from rpent.utils.logging import get_logger logger = get_logger("human_in_the_loop") +@dataclass(frozen=True) +class OperatorCommand: + """Parsed operator input, without granting any scene or motion permission.""" + + name: str + notes: str = "" + request_id: str | None = None + answer: str = "" + + +def parse_operator_command(line: str) -> OperatorCommand | None: + """Parse the shared terminal/Web commands; ordinary chat returns None.""" + parts = line.strip().split(maxsplit=1) + command = parts[0].lower() if parts else "" + if command == "/succes": + command = "/success" + if command not in { + "/done", + "/continue", + "/success", + "/failure", + "/abort", + "/operator", + }: + return None + notes = parts[1] if len(parts) > 1 else "" + if command == "/operator": + tokens = notes.split(maxsplit=2) + if len(tokens) < 2: + raise ValueError("Usage: /operator [notes]") + return OperatorCommand( + name="operator", + request_id=tokens[0], + answer=tokens[1], + notes=tokens[2] if len(tokens) > 2 else "", + ) + return OperatorCommand(name=command[1:], notes=notes) + + class HumanInTheLoopInput: """Use the existing interactive reader, or read an otherwise unowned TTY. @@ -36,40 +78,62 @@ class HumanInTheLoopInput: """ help_text = """Human-interactive exploration commands: - /done Confirm the pending scene reset. - /continue Continue from a pending operator verdict. - /success Finish successfully and save exploration memory. - /failure Finish with a failure record. - /abort Abort exploration without publishing success memory. + /done [notes] Confirm the pending scene reset, optionally describing the scene. + /continue [notes] Continue from a pending verdict with optional instructions. + /success [notes] Record this successful attempt and continue in a new session. + /failure [notes] Record this failed attempt and continue in a new session. + /abort [notes] Abort exploration without publishing success memory. Words without / remain normal messages to the agent. """ - def __init__(self, *, interactive: bool) -> None: + def __init__(self, *, interactive: bool, feedback_path: Path | None = None) -> None: self.interactive = interactive + self._feedback_path = feedback_path + self._feedback: list[str] = [] self._lock = threading.Lock() self._pending: tuple[str, queue.Queue[str | None]] | None = None self._pending_kind: str | None = None self._closed = False - self._verdict_handler: Callable[[str], bool] | None = None + self._continue_revision = 0 + self._verdict_handler: Callable[[str, str], bool] | None = None - def bind_verdict(self, handler: Callable[[str], bool] | None) -> None: + def bind_verdict(self, handler: Callable[[str, str], bool] | None) -> None: """Bind program-level verdict control for the active exploration session.""" with self._lock: self._verdict_handler = handler def route_line(self, line: str) -> bool: """Consume operator commands; return False for ordinary planner input.""" - command = line.strip().lower() + try: + parsed = parse_operator_command(line) + except ValueError as exc: + logger.warning("%s", exc) + return True + if parsed is None: + if line.strip() and not line.startswith("/"): + with self._lock: + self._record_feedback(line.strip()) + return False + command = f"/{parsed.name}" + notes = parsed.notes if command in {"/done", "/continue"}: expected = "reset" if command == "/done" else "verdict" with self._lock: + if command == "/continue" and self._pending is None: + if notes: + self._record_feedback(notes) + return False if self._pending is None or self._pending_kind != expected: logger.warning( "%s refused: no pending %s request.", command, expected ) else: - self._pending[1].put(command[1:]) + if command == "/continue": + self._continue_revision += 1 + self._pending[1].put(command[1:] + (" " + notes if notes else "")) + if notes: + self._record_feedback(notes) self._pending = None self._pending_kind = None logger.info("%s accepted.", command) @@ -78,27 +142,56 @@ def route_line(self, line: str) -> bool: verdict = command[1:] with self._lock: handler = self._verdict_handler - if handler is None or not handler(verdict): + if handler is None or not handler(verdict, notes): logger.warning( "/%s refused: no eligible exploration attempt, or a verdict is already closing the run.", verdict, ) + elif notes: + with self._lock: + self._record_feedback(notes) return True - if line.split(maxsplit=1)[:1] != ["/operator"]: - return False - parts = line.split(maxsplit=2) with self._lock: pending = self._pending - if pending is None or len(parts) != 3 or parts[1] != pending[0]: + if pending is None or parsed.request_id != pending[0]: logger.warning( "No matching operator request; use /operator ." ) else: - pending[1].put(parts[2]) + if self._pending_kind == "verdict" and parsed.answer == "continue": + self._continue_revision += 1 + reply = parsed.answer + (" " + notes if notes else "") + pending[1].put(reply) + self._record_feedback(reply) self._pending = None self._pending_kind = None return True + @property + def continue_revision(self) -> int: + """Count accepted continue replies, including replies during a model turn.""" + with self._lock: + return self._continue_revision + + def _record_feedback(self, text: str) -> None: + self._feedback.append(text) + if self._feedback_path is not None: + self._feedback_path.parent.mkdir(parents=True, exist_ok=True) + with self._feedback_path.open("a") as f: + f.write(json.dumps({"text": text}, ensure_ascii=False) + "\n") + logger.info("[feedback] recorded for exploration handoff: %s", text) + + def feedback_prompt(self) -> str: + with self._lock: + if not self._feedback: + return "" + return ( + "\nOperator feedback, chronological (later task corrections supersede " + "earlier task assumptions, not hardware safety requirements):\n" + + "\n".join(self._feedback) + + "\nAcknowledge the latest exploration focus and adapt your plan.\n" + ) + def close(self) -> None: """Release pending input and detach the active verdict handler.""" with self._lock: @@ -109,6 +202,24 @@ def close(self) -> None: self._pending[1].put(None) self._pending = None + @property + def pending_kind(self) -> str | None: + with self._lock: + return self._pending_kind + + @property + def closed(self) -> bool: + with self._lock: + return self._closed + + def cancel_pending(self) -> None: + """Retire the old request, never confirm it or close the input reader.""" + with self._lock: + if self._pending is not None: + self._pending[1].put(None) + self._pending = None + self._pending_kind = None + def __call__(self, prompt: str, check_cancelled: Callable[[], None]) -> str | None: return self.request(prompt, check_cancelled) @@ -145,12 +256,15 @@ def request( "/done" if kind == "reset" else "/continue" if kind == "verdict" else None ) hint = ( - f"\nShortcut: {shortcut}; /success, /failure or /abort ends exploration." + f"\nShortcut: {shortcut}; /success or /failure starts a new session; " + "/abort ends exploration." if shortcut else "" ) - print(f"\n{prompt}\nReply: /operator {request_id} {hint}", flush=True) try: + print( + f"\n{prompt}\nReply: /operator {request_id} {hint}", flush=True + ) while True: check_cancelled() try: diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 2d66a1765..f0b7cc1ad 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -337,6 +337,11 @@ def cancel_active_and_wait(self) -> None: """Pause admission, cancel queued and active calls, and wait for cleanup.""" self._scheduler.wait(self._scheduler.cancel()) + def wait_active(self) -> None: + """Wait for admitted calls to complete without cancelling them.""" + with self._scheduler.condition: + self._scheduler.condition.wait_for(lambda: not self._scheduler.active) + def resume_calls(self) -> None: """Reopen admission after interrupted calls and their workers have drained.""" self._scheduler.resume() diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index 970a01595..46f6e6b84 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -25,6 +25,7 @@ from robots.dual_franka.toolkit import DualFrankaToolkit from rpent.dashboard.events import NullDashboardEventSink from rpent.memory import MemoryManager +from rpent.planner.base import PlannerResult from rpent.tools import ToolResult @@ -172,10 +173,22 @@ def fail(): assert not env.moves -def test_operator_abort_allows_finish_even_with_budget_and_never_succeeds(setup): +@pytest.mark.parametrize( + "response", + [ + "abort", + "abort calibration board still attached", + " ABORT operator stop ", + None, + ], +) +def test_operator_abort_allows_finish_even_with_budget_and_never_succeeds( + setup, response +): t, env, replies = setup - replies.append("abort") - call(t, "request_scene_reset", reason="initial") + replies.append(response) + reset_result = call(t, "request_scene_reset", reason="initial") + assert reset_result["operator_aborted"] result = call(t, "finish", status="success", summary="stop") assert result["_finish"] and result["operator_aborted"] assert result["status"] == "failure" and not t.solved() and env.resets == 0 @@ -294,22 +307,20 @@ def test_cli_two_sessions_operator_feedback_and_memory_pipeline( from types import SimpleNamespace from rpent.cli import main as cli + from rpent.tools.human_in_the_loop import HumanInTheLoopInput env = FakeEnv() runtimes = [] planners = [] replies = iter(["done", "failure dropped", "done", "success lifted"]) - class Operator: - def __init__(self, **kwargs): - assert kwargs["interactive"] is False - - def __call__(self, prompt, cancelled): + class Operator(HumanInTheLoopInput): + def request(self, prompt, cancelled, *, kind=None): cancelled() return next(replies) - def close(self): - pass + def __call__(self, prompt, cancelled): + return self.request(prompt, cancelled) class Planner: def __init__(self, *args, **kwargs): @@ -368,6 +379,11 @@ def init_runtime(*args): spec = replace(robot_spec.get_robot_spec(), init_runtime=init_runtime) monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) monkeypatch.setattr(cli, "build_planner", Planner) + monkeypatch.setattr( + cli, + "start_interactive_reader", + lambda inputs, **kwargs: inputs.put(kwargs["first_prompt_default"]), + ) monkeypatch.setattr("rpent.tools.human_in_the_loop.HumanInTheLoopInput", Operator) monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True)) monkeypatch.setattr( @@ -377,6 +393,8 @@ def init_runtime(*args): "rpent", "--robot", "dual_franka", + "--planner", + "codex", "--explore", "--explore-sessions", "2", @@ -530,7 +548,7 @@ def active_motion(**kwargs): worker.start() assert entered.wait(2) assert t.request_direct_verdict("success") - assert t.request_direct_verdict("success") # idempotent, never another attempt + assert not t.request_direct_verdict("success") # no duplicate planner EOF assert call(t, "open_gripper", arm="left")["motion_refused"] release.set() worker.join(2) @@ -565,104 +583,111 @@ def test_direct_success_with_failed_observation_does_not_publish(setup): assert t.write_recipe("dual_franka_t0") is None -@pytest.mark.parametrize("robot_name", ["dual_franka", "libero"]) -@pytest.mark.parametrize("verdict", ["success", "failure", "abort"]) -@pytest.mark.parametrize("planner_error", [None, "planner transport failed"]) +@pytest.mark.parametrize( + "verdict,auto_merge_memory,planner_error,session_count,consume_cancel", + [ + ("success", True, None, 2, True), + ("success", False, None, 1, False), + ("success", True, "planner transport failed", 1, True), + ("failure", True, None, 2, False), + ("failure", True, "planner transport failed", 2, True), + ("abort", True, None, 1, False), + ("abort", True, "planner transport failed", 1, True), + ], +) def test_cli_direct_verdict_finalizes_and_merges_only_without_errors( tmp_path, - monkeypatch, - dual_franka_robot_config, + run_interactive_cli, verdict, planner_error, - robot_name, + session_count, + consume_cancel, + auto_merge_memory, ): - import sys - from dataclasses import replace - from types import SimpleNamespace - - from rpent.cli import main as cli - env = FakeEnv() handlers = {} + solve_calls = [] def reader(input_queue, **kwargs): handlers["line"] = kwargs["line_handler"] + handlers["inputs"] = input_queue input_queue.put("test task") - monkeypatch.setattr(cli, "start_interactive_reader", reader) - class Planner: def solve(self, *, toolkit, input_queue, **kwargs): + import queue + + with pytest.raises(queue.Empty): + input_queue.get(block=False) + if solve_calls: + assert "write a grounded outcome summary" in kwargs["user_message"] + assert "Focus on the later task stages" in kwargs["user_message"] + assert not toolkit._scene_ready + if verdict == "success" and auto_merge_memory: + assert ( + tmp_path / "memory/task-specific/dual_franka_t0_recipe.jsonl" + ).is_file() + solve_calls.append(True) + assert not handlers["line"]("Focus on the later task stages") toolkit._operator_input = lambda *args: "done" call(toolkit, "request_scene_reset", reason="test") call(toolkit, "move_delta", arm="right", delta_xyz=[0.01, 0, 0]) - assert handlers["line"]("/" + verdict) - assert input_queue.get(timeout=1) is None - return SimpleNamespace( - finish_result=None, messages=[], stats={}, error=planner_error - ) - - spec = replace( - robot_spec.get_robot_spec(), - init_runtime=lambda *args: ( - [], - { - "env": env, - "model": None, - "task_description": "test", - }, - ), + assert handlers["line"]("/" + verdict + " 建议采用混合控制") + assert handlers["line"]("/" + verdict + " 重复提交") + if consume_cancel: + assert input_queue.get(timeout=1) is None + assert handlers["inputs"].empty() + return PlannerResult(error=planner_error) + + result = run_interactive_cli( + Planner(), + env, + reader, + sessions=session_count, + auto_merge_memory=auto_merge_memory, ) - monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) - original_toolkit_factory = cli.get_toolkit - monkeypatch.setattr( - cli, - "get_toolkit", - lambda name, **kwargs: original_toolkit_factory("dual_franka", **kwargs), - ) - monkeypatch.setattr(cli, "build_planner", lambda *args, **kwargs: Planner()) - monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True)) - monkeypatch.setattr( - sys, - "argv", - [ - "rpent", - "--robot", - robot_name, - "--planner", - "codex", - "--explore", - "--interactive", - "--output-dir", - str(tmp_path / "run"), - "--memory-dir", - str(tmp_path / "memory"), - "--robot-config", - str(dual_franka_robot_config), - ], + assert result == (1 if planner_error and verdict != "failure" else 0) + if verdict == "failure": + failures = list((tmp_path / "memory/_internal/inbox").rglob("*-failure.json")) + assert len(failures) == session_count + assert json.loads(failures[0].read_text())["verdict"]["status"] == "failure" + assert ( + json.loads(failures[0].read_text())["verdict"]["operator_notes"] + == "建议采用混合控制" + ) + assert len(solve_calls) == ( + session_count + if verdict == "failure" or (verdict == "success" and not planner_error) + else 1 ) - assert cli.main() == (1 if planner_error else 0) + if verdict == "success" and not planner_error: + assert ( + len(list((tmp_path / "run/sessions").glob("*/dual_franka_t0_recipe.jsonl"))) + == session_count + ) assert (tmp_path / "memory/task-specific/dual_franka_t0.json").is_file() == ( - verdict == "success" and planner_error is None + auto_merge_memory and verdict == "success" and planner_error is None ) assert ( tmp_path / "memory/task-specific/dual_franka_t0_recipe.jsonl" - ).is_file() == (verdict == "success" and planner_error is None) + ).is_file() == ( + auto_merge_memory and verdict == "success" and planner_error is None + ) events = json.loads( (tmp_path / "run/sessions/session_001/operator_events.json").read_text() ) assert events[-1]["status"] == ("failure" if verdict == "abort" else verdict) - assert events[-1]["operator_finished"] is True + assert events[-1]["operator_finished"] is (verdict == "abort") def test_direct_failure_ends_without_success_memory_even_before_reset(setup): t, env, _ = setup assert t.request_direct_verdict("failure") - assert t.request_direct_verdict("failure") + assert not t.request_direct_verdict("failure") assert not t.request_direct_verdict("success") assert call(t, "request_scene_reset", reason="late reset")["motion_refused"] result = t.finalize_direct_verdict() - assert result["status"] == "failure" and result["operator_finished"] + assert result["status"] == "failure" and not result["operator_finished"] assert not t.solved() and env.resets == 0 assert t.write_recipe("dual_franka_t0") is None @@ -726,3 +751,532 @@ def observe(): assert published[-1].command["action"] == "move_delta" assert len(env.moves) == 1 and env.moves[0][0] == "left" np.testing.assert_allclose(env.moves[0][1], [0.01, 0, 0]) + + +@pytest.fixture +def run_interactive_cli(tmp_path, monkeypatch, dual_franka_robot_config): + import sys + from dataclasses import replace + from types import SimpleNamespace + + from rpent.cli import main as cli + + monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True)) + + def run( + planner, + env, + reader, + *, + sessions, + auto_merge_memory=True, + ): + spec = replace( + robot_spec.get_robot_spec(), + init_runtime=lambda *a: ( + [], + {"env": env, "model": None, "task_description": "test"}, + ), + ) + monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) + monkeypatch.setattr(cli, "start_interactive_reader", reader) + monkeypatch.setattr(cli, "build_planner", lambda *a, **kw: planner) + monkeypatch.setattr( + sys, + "argv", + [ + "rpent", + "--robot", + "dual_franka", + "--planner", + "codex", + "--explore", + "--interactive", + "--explore-sessions", + str(sessions), + "--output-dir", + str(tmp_path / "run"), + "--memory-dir", + str(tmp_path / "memory"), + "--robot-config", + str(dual_franka_robot_config), + "--auto-merge-memory" + if auto_merge_memory + else "--no-auto-merge-memory", + ], + ) + return cli.main() + + return run + + +@pytest.mark.parametrize("last_verdict", ["failure", "abort", "unfinished"]) +def test_success_memory_survives_a_later_unsuccessful_session( + tmp_path, monkeypatch, last_verdict, run_interactive_cli +): + env = FakeEnv() + handlers = {} + calls = [] + waiting_tools = [] + + def reader(input_queue, **kwargs): + handlers["line"] = kwargs["line_handler"] + input_queue.put("test task") + + class Planner: + def solve(self, *, toolkit, input_queue, **kwargs): + assert not toolkit._scene_ready + if calls: + assert (tmp_path / "memory/task-specific/dual_franka_t0.json").is_file() + verdict = "success" if not calls else last_verdict + calls.append(verdict) + if verdict == "unfinished": + import threading + + ready = threading.Event() + monkeypatch.setattr( + "builtins.print", lambda *args, **kwargs: ready.set() + ) + worker = threading.Thread( + target=lambda: call( + toolkit, "request_scene_reset", reason="pending" + ) + ) + waiting_tools.append(worker) + worker.start() + assert ready.wait(2) + assert handlers["line"]("/abort") + return PlannerResult() + toolkit._operator_input = lambda *args: "done" + call(toolkit, "request_scene_reset", reason="test") + call(toolkit, "move_delta", arm="right", delta_xyz=[0.01, 0, 0]) + assert handlers["line"]("/" + verdict) + assert input_queue.get(timeout=1) is None + return PlannerResult() + + sessions = 50 if last_verdict == "unfinished" else 2 + assert run_interactive_cli(Planner(), env, reader, sessions=sessions) == 0 + assert calls == ["success", last_verdict] + published = json.loads( + (tmp_path / "memory/task-specific/dual_franka_t0.json").read_text() + ) + assert "session_001" in published["state_trace"] + assert not ( + tmp_path / "run/sessions/session_002/dual_franka_t0_recipe.jsonl" + ).exists() + assert env.resets == (1 if last_verdict == "unfinished" else 2) + for worker in waiting_tools: + worker.join(2) + assert not worker.is_alive() + + +@pytest.mark.parametrize("kind", ["reset", "verdict"]) +def test_close_retires_waiting_request_without_robot_motion(setup, monkeypatch, kind): + import threading + + from rpent.tools.human_in_the_loop import HumanInTheLoopInput + + t, env, replies = setup + if kind == "verdict": + reset(t, replies) + resets_before = env.resets + broker = HumanInTheLoopInput(interactive=True) + t._operator_input = broker + ready = threading.Event() + monkeypatch.setattr("builtins.print", lambda *args, **kwargs: ready.set()) + result = [] + tool = "request_scene_reset" if kind == "reset" else "request_operator_verdict" + kwargs = {"reason": "test cancellation"} if kind == "reset" else {} + worker = threading.Thread(target=lambda: result.append(call(t, tool, **kwargs))) + worker.start() + assert ready.wait(2) + old_id = broker._pending[0] + t.close() + worker.join(2) + assert not worker.is_alive() + assert broker.pending_kind is None + assert env.resets == resets_before and not env.moves + assert call(t, "request_scene_reset", reason="late call")["motion_refused"] + assert broker.route_line(f"/operator {old_id} done") + assert broker.pending_kind is None + + +@pytest.mark.parametrize("pending_reset", [False, True]) +def test_attended_reply_resumes_same_toolkit_without_automatic_reset( + setup, monkeypatch, tmp_path, pending_reset +): + import queue + import threading + + from rpent.cli.attended import _solve_attended + from rpent.tools.human_in_the_loop import HumanInTheLoopInput + + t, env, replies = setup + if not pending_reset: + reset(t, replies) + broker = HumanInTheLoopInput(interactive=True) + t._operator_input = broker + assert not broker.route_line("Keep the completed task stages") + inputs = queue.Queue() + first_reply = threading.Event() + second_reply = threading.Event() + prompt_ready = threading.Event() + monkeypatch.setattr("builtins.print", lambda *a, **kw: prompt_ready.set()) + workers = [] + calls = [] + servers = [] + + async def reset_over_http(url): + import httpx + from mcp import ClientSession + from mcp.client.streamable_http import streamable_http_client + + async with httpx.AsyncClient(trust_env=False) as client: + async with streamable_http_client(url, http_client=client) as ( + read, + write, + _, + ): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.call_tool( + "request_scene_reset", {"reason": "test"} + ) + assert not result.isError + + class Planner: + def solve(self, *, toolkit, user_message, **kwargs): + assert toolkit is t + server = kwargs["mcp_server"] + servers.append(server) + assert server._thread.is_alive() and not server._server.should_exit + calls.append(user_message) + if len(calls) == 1: + if pending_reset: + import asyncio + + worker = threading.Thread( + target=lambda: asyncio.run(reset_over_http(server.url)) + ) + worker.start() + workers.append(worker) + assert prompt_ready.wait(2) + first_reply.set() + return PlannerResult( + messages=[{"role": "assistant", "content": "Please confirm"}] + ) + assert "SAME attempt" in user_message + assert user_message.count("Operator feedback, chronological") == 1 + assert "Keep the completed task stages" in user_message + assert t._scene_ready + assert env.resets == 1 + t.request_direct_verdict("success", "test complete") + second_reply.set() + return PlannerResult() + + results = [] + runner = threading.Thread( + target=lambda: results.append( + _solve_attended( + Planner(), + operator_input=broker, + state_output_dir=tmp_path / "session", + keep_mcp_alive=True, + toolkit=t, + input_queue=inputs, + system_prompt="test", + user_message="original task" + broker.feedback_prompt(), + max_turns=10, + ) + ) + ) + runner.start() + assert first_reply.wait(2) + assert not second_reply.wait(0.15) + if pending_reset: + assert broker.route_line("/done 保留前半段成果") + else: + inputs.put("场景已调整，请继续后半段") + runner.join(3) + for worker in workers: + worker.join(2) + assert not runner.is_alive() and len(results) == 1 + assert len(calls) == 2 and env.resets == 1 + assert len(list((tmp_path / "session").glob("dialogue_*.json"))) == 2 + assert not env.moves + assert servers[0] is servers[1] + assert servers[0]._thread is None + + +@pytest.mark.parametrize("input_closed", [False, True]) +def test_attended_wait_eof_aborts_without_another_planner_call( + setup, tmp_path, monkeypatch, input_closed +): + import queue + + from rpent.cli.attended import _solve_attended + from rpent.tools.human_in_the_loop import HumanInTheLoopInput + + t, env, _ = setup + inputs = queue.Queue() + inputs.put(None) + calls = [] + broker = HumanInTheLoopInput(interactive=True) + if input_closed: + broker.close() + monkeypatch.setattr( + t, + "wait_active", + lambda: pytest.fail("closed input must abort before waiting"), + ) + + class Planner: + def solve(self, **kwargs): + calls.append(True) + return PlannerResult() + + _solve_attended( + Planner(), + operator_input=broker, + state_output_dir=tmp_path / "session", + toolkit=t, + input_queue=inputs, + user_message="task", + system_prompt="", + max_turns=10, + ) + assert calls == [True] + assert t.finalize_direct_verdict()["operator_aborted"] + assert not env.resets and not env.moves + + +@pytest.mark.parametrize("reply_style", ["shortcut", "request_id"]) +def test_attended_consumes_continue_once_without_duplicate_wait( + setup, tmp_path, monkeypatch, reply_style +): + import queue + import threading + + from rpent.cli.attended import _solve_attended + from rpent.tools.human_in_the_loop import HumanInTheLoopInput + + t, env, _ = setup + broker = HumanInTheLoopInput(interactive=True) + ready = threading.Event() + monkeypatch.setattr("builtins.print", lambda *a, **kw: ready.set()) + calls = [] + inputs = queue.Queue() + inputs.put(None) + + class Planner: + def solve(self, **kwargs): + calls.append(kwargs["user_message"]) + if len(calls) == 1: + replies = [] + thread = threading.Thread( + target=lambda: replies.append( + broker.request("continue?", lambda: None, kind="verdict") + ) + ) + thread.start() + assert ready.wait(2) + command = ( + "/continue" + if reply_style == "shortcut" + else f"/operator {broker._pending[0]} continue" + ) + assert broker.route_line(command) + thread.join(2) + assert replies == ["continue"] + return PlannerResult() + + _solve_attended( + Planner(), + operator_input=broker, + state_output_dir=tmp_path / "session", + toolkit=t, + input_queue=inputs, + user_message="task", + system_prompt="", + max_turns=10, + ) + assert len(calls) == 2 + assert "already answered continue" in calls[1] + assert broker.continue_revision == 1 + assert t.finalize_direct_verdict()["operator_aborted"] + assert not env.resets and not env.moves + + +@pytest.mark.parametrize("phase", ["active", "reply_returned", "operator_request"]) +def test_attended_verdict_preserves_next_attempt_input( + setup, tmp_path, monkeypatch, phase +): + import queue + import threading + + from rpent.cli import attended + from rpent.planner.utils.http_mcp_server import HttpMcpServer + from rpent.session.input import SessionInputQueue + from rpent.tools.human_in_the_loop import HumanInTheLoopInput + from tests.unit_tests.rpent.planner.test_codex_contracts import ( + FakeCodex, + FakeTurn, + RecordingSink, + install_fake_backend, + make_planner, + ) + + install_fake_backend(monkeypatch) + monkeypatch.setattr("rpent.planner.codex.HttpMcpServer", HttpMcpServer) + toolkit, env, replies = setup + reset(toolkit, replies) + inputs = queue.Queue() + scope = SessionInputQueue(inputs) + broker = HumanInTheLoopInput(interactive=True) + ready, interrupted = threading.Event(), threading.Event() + + def verdict(outcome, notes): + assert toolkit.request_direct_verdict(outcome, notes) + scope.cancel() + broker.cancel_pending() + return True + + broker.bind_verdict(verdict) + request = None + if phase == "operator_request": + request_ready = threading.Event() + monkeypatch.setattr("builtins.print", lambda *a, **kw: request_ready.set()) + request = threading.Thread( + target=lambda: broker.request("Evaluate?", lambda: None, kind="verdict") + ) + request.start() + assert request_ready.wait(3) + + def info(message, *args): + if message.startswith("Waiting for"): + ready.set() + + monkeypatch.setattr(attended.logger, "info", info) + + def interrupt(turn): + turn.interrupt_calls += 1 + interrupted.set() + + def stream(turn): + if phase == "active": + ready.set() + assert interrupted.wait(3) + yield from turn.events + + monkeypatch.setattr(FakeTurn, "interrupt", interrupt) + monkeypatch.setattr(FakeTurn, "stream", stream) + planner = make_planner(tmp_path, RecordingSink(), timeout_s=5) + results = [] + runner = threading.Thread( + target=lambda: results.append( + attended._solve_attended( + planner, + operator_input=broker, + state_output_dir=tmp_path / "attempt1", + keep_mcp_alive=True, + toolkit=toolkit, + input_queue=scope, + system_prompt="test", + user_message="task", + max_turns=10, + ) + ) + ) + runner.start() + try: + assert ready.wait(3) + assert broker.route_line("/success") + inputs.put("Focus on the later stages") + runner.join(5) + assert not runner.is_alive() and results[0].error is None + assert FakeCodex.instances[0].thread.fake_turn.interrupt_calls == int( + phase == "active" + ) + finally: + scope.cancel() + broker.close() + interrupted.set() + runner.join(5) + if request is not None: + request.join(3) + assert not request.is_alive() + + delivered = threading.Event() + + def steer(turn, text): + turn.steered.append(text) + delivered.set() + + def next_stream(turn): + assert delivered.wait(3) + yield from turn.events + + monkeypatch.setattr(FakeTurn, "steer", steer) + monkeypatch.setattr(FakeTurn, "stream", next_stream) + next_toolkit = DualFrankaToolkit( + runtime_kwargs={"env": env, "model": None, "task_description": "test"}, + dashboard_events=NullDashboardEventSink(), + memory=toolkit.memory, + mode="exploration", + state_output_dir=tmp_path / "attempt2", + operator_input=broker, + ) + result = planner.solve( + system_prompt="test", + user_message="next attempt", + toolkit=next_toolkit, + max_turns=10, + input_queue=SessionInputQueue(inputs), + ) + assert result.error is None + turn = FakeCodex.instances[-1].thread.fake_turn + assert turn.steered == ["Focus on the later stages"] and turn.interrupt_calls == 0 + assert inputs.empty() + + +def test_continue_captures_scene_after_operator_adjustment(setup, monkeypatch): + t, env, replies = setup + reset(t, replies) + original = env.get_observation + scene = [10] + reads = [] + published = [] + monkeypatch.setattr(t, "_publish_step", published.append) + + def observe(): + reads.append(scene[0]) + obs = original() + obs["d455_images"][:] = scene[0] + return obs + + env.get_observation = observe + + def operator(*args): + scene[0] = 90 + return "continue moved object" + + t._operator_input = operator + observed = t.execute_tool("request_operator_verdict", {}) + result = observed.data + assert result["status"] == "continue" + assert np.all(t.state.load("d455.png", step=result["evidence_step"]) == 10) + assert np.all(t.state.load("d455.png", step=result["observation_step"]) == 90) + assert observed.images == [ + t.state.load_bytes("d455.png", step=result["observation_step"]) + ] + assert result["observation_step"] == result["evidence_step"] + 1 + assert reads == [10, 90] + assert [record.step_idx for record in published] == [ + result["evidence_step"], + result["observation_step"], + ] + assert [record.command for record in published] == [ + {"action": "observe_for_verdict"}, + {"action": "observe_current"}, + ] + assert env.resets == 1 and not env.moves diff --git a/tests/unit_tests/rpent/cli/test_main_contracts.py b/tests/unit_tests/rpent/cli/test_main_contracts.py index e33ef3357..16702e92c 100644 --- a/tests/unit_tests/rpent/cli/test_main_contracts.py +++ b/tests/unit_tests/rpent/cli/test_main_contracts.py @@ -111,6 +111,9 @@ def test_public_cli_help_runs_from_source_checkout_without_a_robot_runtime() -> "--interactive", ): assert option in result.stdout + assert "--dashboard-language" in result.stdout + assert "--dashboard-record" not in result.stdout + assert "--dashboard-camera-stream" not in result.stdout @pytest.mark.parametrize( @@ -185,6 +188,21 @@ def test_robot_and_env_aliases_are_mutually_exclusive( @pytest.mark.parametrize( ("argv", "message"), [ + ( + ["--robot", "libero", "--planner", "api", "--reasoning-effort", "ultra"], + "requires --planner codex", + ), + ( + [ + "--robot", + "libero", + "--planner", + "claude_code", + "--reasoning-effort", + "ultra", + ], + "requires --planner codex", + ), ( ["--robot", "libero", "--dashboard", "--interactive"], "cannot be used together", @@ -356,6 +374,7 @@ def test_real_robot_terminal_requirement_fails_before_runtime( spec = SimpleNamespace( is_real_robot=True, dashboard=None, + supports_human_interactive_exploration=False, add_cli_args=lambda parser, use_dashboard: None, ) monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) @@ -784,3 +803,24 @@ def finalize_run(context: RunFinalizationContext) -> Path: } assert robot_toolkit.closed is True assert daemon.stopped is True + + +def test_codex_ultra_effort_reaches_robot_config_parser( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, args = _capture_validated_args( + monkeypatch, + [ + "--robot", + "libero", + "--planner", + "codex", + "--model", + "gpt-6-astra", + "--reasoning-effort", + "ultra", + ], + ) + + assert args.model == "gpt-6-astra" + assert args.reasoning_effort == "ultra" diff --git a/tests/unit_tests/rpent/cli/test_tui_contracts.py b/tests/unit_tests/rpent/cli/test_tui_contracts.py new file mode 100644 index 000000000..481d83aae --- /dev/null +++ b/tests/unit_tests/rpent/cli/test_tui_contracts.py @@ -0,0 +1,111 @@ +# Copyright 2026 The RPent Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import queue +import threading + +import pytest + +from rpent.cli.tui import initial_user_message, next_user_line +from rpent.session.input import SessionInputQueue + + +@pytest.mark.parametrize("read_line", [initial_user_message, next_user_line]) +@pytest.mark.parametrize("end_of_input", [None, "/quit", "/exit", "/q"]) +def test_scoped_input_preserves_feedback_and_real_exit(read_line, end_of_input): + source = queue.Queue() + scope = SessionInputQueue(source) + scope.put(" ") + scope.put(" inspect the other bowl ") + scope.put(end_of_input) + + assert read_line(scope) == "inspect the other bowl" + assert read_line(scope) is None + assert source.empty() + + +def test_cancelled_scope_leaves_feedback_and_real_eof_for_next_scope(): + source = queue.Queue() + previous = SessionInputQueue(source) + previous.cancel() + source.put("next attempt feedback") + source.put(None) + + assert previous.cancelled + assert previous.get(block=False) is None + assert previous.get(timeout=0) is None + current = SessionInputQueue(source) + assert not current.cancelled + assert current.get(block=False) == "next attempt feedback" + assert current.get(block=False) is None + assert source.empty() + + +def test_nested_scope_cleanup_does_not_cancel_attended_attempt(): + source = queue.Queue() + attempt = SessionInputQueue(source) + previous_reply = SessionInputQueue(attempt) + previous_reply.cancel() + attempt.put("continue after that reply") + + assert previous_reply.get() is None + assert not attempt.cancelled + current_reply = SessionInputQueue(attempt) + assert next_user_line(current_reply) == "continue after that reply" + attempt.cancel() + assert current_reply.get() is None + assert source.empty() + + +def test_scoped_input_retains_queue_nonblocking_and_timeout_contract(): + scope = SessionInputQueue(queue.Queue()) + with pytest.raises(queue.Empty): + scope.get(block=False) + with pytest.raises(queue.Empty): + scope.get(timeout=0) + with pytest.raises(queue.Empty): + scope.get(timeout=0.01) + with pytest.raises(ValueError, match="non-negative"): + scope.get(timeout=-1) + scope.put("available immediately") + assert scope.get(timeout=0) == "available immediately" + + +def test_cancel_retires_waiting_reader_without_consuming_later_input(): + reading = threading.Event() + + class ObservedQueue(queue.Queue): + def get(self, block=True, timeout=None): + reading.set() + return super().get(block=block, timeout=timeout) + + source = ObservedQueue() + scope = SessionInputQueue(source) + received = [] + reader = threading.Thread(target=lambda: received.append(scope.get()), daemon=True) + reader.start() + try: + assert reading.wait(1) + scope.cancel() + source.put("new attempt") + reader.join(timeout=1) + assert not reader.is_alive() + assert received == [None] + assert source.get(block=False) == "new attempt" + assert source.empty() + finally: + scope.cancel() + reader.join(timeout=1) diff --git a/tests/unit_tests/rpent/evaluation/test_result.py b/tests/unit_tests/rpent/evaluation/test_result.py index f43b031fa..8d39b20d7 100644 --- a/tests/unit_tests/rpent/evaluation/test_result.py +++ b/tests/unit_tests/rpent/evaluation/test_result.py @@ -21,6 +21,34 @@ import pytest from rpent.evaluation import write_json_atomic +from rpent.evaluation.result import record_operator_outcome + + +@pytest.mark.parametrize("outcome", ["success", "failure"]) +def test_operator_outcome_preserves_evidence_and_refuses_overwrite(tmp_path, outcome): + verdict = {"operator_verdict": outcome, "notes": "operator evidence"} + messages = [{"role": "user", "content": "keep the completed part"}] + kwargs = { + "memory_root": tmp_path, + "recipe_tag": "dual_franka_t5", + "run_name": "run-1", + "session_number": 2, + "state_output_dir": tmp_path / "session_002", + "verdict": verdict, + "messages": messages, + "planner_error": "interrupted by verdict", + } + path = record_operator_outcome(**kwargs) + assert path.name == f"run-1-session-002-{outcome}.json" + record = json.loads(path.read_text()) + assert record["verdict"] == verdict + assert record["messages"] == messages + assert record["session"] == 2 + assert record["evidence_dir"] == str(tmp_path / "session_002") + assert record["planner_error"] == "interrupted by verdict" + with pytest.raises(FileExistsError): + record_operator_outcome(**kwargs) + assert json.loads(path.read_text()) == record def test_write_json_atomic_replaces_complete_record(tmp_path: Path) -> None: diff --git a/tests/unit_tests/rpent/planner/test_claude_contracts.py b/tests/unit_tests/rpent/planner/test_claude_contracts.py index 7b3a93434..657be3165 100644 --- a/tests/unit_tests/rpent/planner/test_claude_contracts.py +++ b/tests/unit_tests/rpent/planner/test_claude_contracts.py @@ -17,20 +17,24 @@ import asyncio import json import queue +import threading from pathlib import Path from typing import Any import claude_agent_sdk import pytest +from rpent.cli.tui import next_user_line from rpent.dashboard.events import TranscriptEvent, UsageEvent from rpent.planner.claude_code import ( ClaudeCodePlanner, _build_rpent_server, _ClaudeSessionDriver, _Recorder, + _TerminalSessionAdapter, _tool_result_to_mcp, ) +from rpent.session.input import SessionInputQueue from rpent.tools import ToolResult, tool from rpent.tools.common import CommonTools @@ -583,3 +587,92 @@ async def close(self): await asyncio.gather(running, return_exceptions=True) asyncio.run(scenario()) + + +@pytest.fixture(autouse=True) +def tool_template_output_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("rpent.utils.templates.get_output_dir", lambda: tmp_path) + + +def test_terminal_adapter_close_does_not_stop_the_next_solve() -> None: + source = queue.Queue() + emitted = [] + queries = [] + interrupts = [] + + class Driver: + async def interrupt(self): + interrupts.append(True) + + async def query(self, text): + queries.append(text) + + async def run_two_solvers(): + first = _TerminalSessionAdapter( + toolkit=FakeToolkit(), input_queue=source, emit_user=emitted.append + ) + source.put(None) + await first.run(Driver()) + await first.close() + assert source.empty() + + second = _TerminalSessionAdapter( + toolkit=FakeToolkit(), input_queue=source, emit_user=emitted.append + ) + source.put("inspect the next attempt") + source.put("/quit") + await second.run(Driver()) + await second.close() + + asyncio.run(run_two_solvers()) + + assert emitted == ["inspect the next attempt"] + assert queries == emitted + assert len(interrupts) == 3 + assert source.empty() + + +def test_terminal_adapter_cleanup_retires_delayed_reader( + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = queue.Queue() + attempt = SessionInputQueue(source) + started = threading.Event() + release = threading.Event() + emitted = [] + queries = [] + + def delayed_read(input_queue): + started.set() + assert release.wait(1) + return next_user_line(input_queue) + + monkeypatch.setattr("rpent.planner.claude_code.next_user_line", delayed_read) + + class Driver: + async def interrupt(self): + pass + + async def query(self, text): + queries.append(text) + + async def run(): + adapter = _TerminalSessionAdapter( + toolkit=FakeToolkit(), input_queue=attempt, emit_user=emitted.append + ) + pump = asyncio.create_task(adapter.run(Driver())) + try: + assert await asyncio.to_thread(started.wait, 1) + source.put("feedback for the next reply") + await adapter.close() + finally: + release.set() + await asyncio.wait_for(pump, timeout=1) + + asyncio.run(run()) + + assert emitted == [] + assert queries == [] + assert not attempt.cancelled + assert attempt.get(block=False) == "feedback for the next reply" + assert source.empty() diff --git a/tests/unit_tests/rpent/planner/test_codex_contracts.py b/tests/unit_tests/rpent/planner/test_codex_contracts.py index c548fae9f..4b8076bc9 100644 --- a/tests/unit_tests/rpent/planner/test_codex_contracts.py +++ b/tests/unit_tests/rpent/planner/test_codex_contracts.py @@ -338,9 +338,11 @@ def codex_config(**kwargs: Any) -> dict[str, Any]: assert os.environ["no_proxy"] == ".corp.invalid" +@pytest.mark.parametrize("effort", ["none", "ultra"]) def test_planner_forwards_configured_service_tier( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + effort: str, ) -> None: install_fake_backend(monkeypatch) monkeypatch.setenv("CODEX_SERVICE_TIER", "fast") @@ -351,7 +353,14 @@ def test_planner_forwards_configured_service_tier( }, ] - make_planner(tmp_path, RecordingSink()).solve( + CodexPlanner( + output_dir=str(tmp_path), + repo_root=tmp_path, + model="gpt-6-astra", + reasoning_effort=effort, + timeout_s=1, + dashboard_events=RecordingSink(), + ).solve( system_prompt="system rules", user_message="user task", toolkit=FakeToolkit(), @@ -359,7 +368,9 @@ def test_planner_forwards_configured_service_tier( ) fake_codex = FakeCodex.instances[0] + assert fake_codex.thread_options["model"] == "gpt-6-astra" assert fake_codex.thread_options["service_tier"] == "fast" + assert fake_codex.thread.turn_prompts[0][1]["effort"].value == effort assert fake_codex.thread.turn_prompts[0][1]["service_tier"] == "fast" @@ -1227,3 +1238,90 @@ def test_retry_error_is_visible_without_poisoning_successful_turn(): ) assert "Retries exhausted" in recorder.error assert "Model connection failed" in str(sink.events[-1]) + + +def test_reply_cleanup_returns_feedback_dequeued_before_cancellation( + tmp_path, monkeypatch +): + from rpent.cli.tui import next_user_line + from rpent.session.input import SessionInputQueue + + install_fake_backend(monkeypatch) + dequeued = threading.Event() + cancelled = threading.Event() + cancel = SessionInputQueue.cancel + + def read(scope): + line = next_user_line(scope) + dequeued.set() + assert cancelled.wait(2) + return line + + def cancel_scope(scope): + cancel(scope) + cancelled.set() + + def stream(turn): + assert dequeued.wait(2) + yield from turn.events + + monkeypatch.setattr(codex_module, "next_user_line", read) + monkeypatch.setattr(SessionInputQueue, "cancel", cancel_scope) + monkeypatch.setattr(FakeTurn, "stream", stream) + inputs = queue.Queue() + inputs.put("Feedback entered as the reply ended") + result = make_planner(tmp_path, RecordingSink()).solve( + system_prompt="test", + user_message="task", + toolkit=FakeToolkit(), + max_turns=10, + input_queue=inputs, + ) + assert result.error is None + turn = FakeCodex.instances[-1].thread.fake_turn + assert turn.steered == [] + assert turn.interrupt_calls == 0 + assert inputs.get_nowait() == "Feedback entered as the reply ended" + assert inputs.empty() + + +@pytest.mark.parametrize("end_input", [None, "/quit", "/exit"]) +def test_terminal_eof_and_quit_interrupt_codex(tmp_path, monkeypatch, end_input): + install_fake_backend(monkeypatch) + interrupted = threading.Event() + + def interrupt(turn): + turn.interrupt_calls += 1 + interrupted.set() + + def stream(turn): + assert interrupted.wait(2) + yield from turn.events + + monkeypatch.setattr(FakeTurn, "interrupt", interrupt) + monkeypatch.setattr(FakeTurn, "stream", stream) + inputs = queue.Queue() + inputs.put(end_input) + result = make_planner(tmp_path, RecordingSink()).solve( + system_prompt="test", + user_message="task", + toolkit=FakeToolkit(), + max_turns=10, + input_queue=inputs, + ) + assert result.error is None + assert FakeCodex.instances[-1].thread.fake_turn.interrupt_calls == 1 + assert inputs.empty() + + +def test_zero_timeout_disables_deadline(tmp_path, monkeypatch): + install_fake_backend(monkeypatch) + planner = make_planner(tmp_path, RecordingSink(), timeout_s=0) + assert planner._timeout_s is None + result = planner.solve( + system_prompt="test", user_message="task", toolkit=FakeToolkit(), max_turns=10 + ) + assert result.error is None + assert FakeCodex.instances[-1].thread.fake_turn.interrupt_calls == 0 + with pytest.raises(ValueError, match="nonnegative"): + make_planner(tmp_path, RecordingSink(), timeout_s=-1) diff --git a/tests/unit_tests/rpent/tools/test_human_in_the_loop.py b/tests/unit_tests/rpent/tools/test_human_in_the_loop.py index 28e5eb2f2..bec717788 100644 --- a/tests/unit_tests/rpent/tools/test_human_in_the_loop.py +++ b/tests/unit_tests/rpent/tools/test_human_in_the_loop.py @@ -18,10 +18,44 @@ import pytest -from rpent.tools.human_in_the_loop import HumanInTheLoopInput +from rpent.tools.human_in_the_loop import ( + HumanInTheLoopInput, + OperatorCommand, + parse_operator_command, +) from rpent.tools.toolkit import ToolCancelled +@pytest.mark.parametrize( + "line,expected", + [ + ("please continue", None), + ("success", None), + ("/rpent-task --task-id 5", None), + (" /DONE scene restored ", OperatorCommand("done", "scene restored")), + ("/continue keep the bowl", OperatorCommand("continue", "keep the bowl")), + ("/succes", OperatorCommand("success")), + ("/failure missed grasp", OperatorCommand("failure", "missed grasp")), + ("/abort", OperatorCommand("abort")), + ( + "/operator request-1 continue keep the bowl", + OperatorCommand("operator", "keep the bowl", "request-1", "continue"), + ), + ], +) +def test_shared_operator_command_parser(line, expected): + assert parse_operator_command(line) == expected + + +@pytest.mark.parametrize("line", ["/operator", "/operator request-1"]) +def test_incomplete_operator_command_never_becomes_chat(line): + with pytest.raises(ValueError, match="Usage: /operator"): + parse_operator_command(line) + broker = HumanInTheLoopInput(interactive=True) + assert broker.route_line(line) + assert broker.feedback_prompt() == "" + + def test_operator_replies_are_request_scoped_and_do_not_consume_steering(monkeypatch): broker = HumanInTheLoopInput(interactive=True) ready = threading.Event() @@ -43,6 +77,31 @@ def test_operator_replies_are_request_scoped_and_do_not_consume_steering(monkeyp assert broker._pending is None +@pytest.mark.parametrize("kind,command", [("reset", "/done"), ("verdict", "/continue")]) +def test_feedback_notes_are_returned_and_preserved( + tmp_path, monkeypatch, kind, command +): + broker = HumanInTheLoopInput( + interactive=True, feedback_path=tmp_path / "feedback.jsonl" + ) + ready = threading.Event() + result = [] + monkeypatch.setattr("builtins.print", lambda *args, **kwargs: ready.set()) + worker = threading.Thread( + target=lambda: result.append(broker.request("confirm", lambda: None, kind=kind)) + ) + worker.start() + assert ready.wait(2) + assert not broker.route_line("保留前半段成果，重点练后半段") + assert not result # Feedback alone never authorizes robot reset. + assert broker.route_line(command + " 篮筐中的碗盘保持不动") + worker.join(2) + assert result == [command[1:] + " 篮筐中的碗盘保持不动"] + assert "重点练后半段" in broker.feedback_prompt() + assert "篮筐中的碗盘保持不动" in broker.feedback_prompt() + assert len((tmp_path / "feedback.jsonl").read_text().splitlines()) == 2 + + def test_operator_eof_and_cancellation_release_pending_requests(monkeypatch): broker = HumanInTheLoopInput(interactive=True) ready = threading.Event() @@ -123,7 +182,7 @@ def route(line): def test_success_is_control_and_never_steering(): broker = HumanInTheLoopInput(interactive=True) calls = [] - broker.bind_verdict(lambda verdict: calls.append(verdict) or True) + broker.bind_verdict(lambda verdict, notes: calls.append(verdict) or True) assert not broker.route_line(" success ") assert broker.route_line("/success") assert len(calls) == 1 @@ -133,16 +192,39 @@ def test_success_is_control_and_never_steering(): assert len(calls) == 1 +@pytest.mark.parametrize("verdict", ["success", "failure", "abort"]) +def test_verdict_notes_are_control_not_steering(tmp_path, verdict): + broker = HumanInTheLoopInput( + interactive=True, feedback_path=tmp_path / "feedback.jsonl" + ) + calls = [] + broker.bind_verdict(lambda value, notes: calls.append((value, notes)) or True) + assert broker.route_line(f"/{verdict} 建议采用混合控制") + assert calls == [(verdict, "建议采用混合控制")] + assert "建议采用混合控制" in broker.feedback_prompt() + + +def test_failed_prompt_display_does_not_leave_pending_request(monkeypatch): + broker = HumanInTheLoopInput(interactive=True) + + def fail(*args, **kwargs): + raise OSError("terminal closed") + + monkeypatch.setattr("builtins.print", fail) + with pytest.raises(OSError, match="terminal closed"): + broker.request("reset", lambda: None, kind="reset") + assert broker.pending_kind is None + + def test_only_slash_verdicts_are_control_commands(): broker = HumanInTheLoopInput(interactive=True) verdicts = [] - broker.bind_verdict(lambda verdict: verdicts.append(verdict) or True) + broker.bind_verdict(lambda verdict, notes: verdicts.append(verdict) or True) for text in ( "success", "failure", "抓取成功了", "success 请继续", - "/success later", ): assert not broker.route_line(text) assert broker.route_line("/success") @@ -164,7 +246,7 @@ def test_shortcuts_only_answer_matching_active_request( ready = threading.Event() values = [] monkeypatch.setattr("builtins.print", lambda *a, **kw: ready.set()) - assert broker.route_line(command) # no buffering before a request + assert broker.route_line(command) is (command != "/continue") ready.clear() thread = threading.Thread( target=lambda: values.append(broker.request("test", lambda: None, kind=kind)) @@ -176,14 +258,14 @@ def test_shortcuts_only_answer_matching_active_request( assert broker.route_line(command) thread.join(2) assert not thread.is_alive() and values == [answer] - assert broker.route_line(command) # duplicate cannot authorize another operation + assert broker.route_line(command) is (command != "/continue") assert not broker.route_line(answer) def test_abort_is_control_but_bare_words_are_chat(): broker = HumanInTheLoopInput(interactive=True) calls = [] - broker.bind_verdict(lambda verdict: calls.append(verdict) or True) + broker.bind_verdict(lambda verdict, notes: calls.append(verdict) or True) assert broker.route_line("/abort") assert calls == ["abort"] for word in ("done", "continue", "abort", "success", "failure"): From ec9cfa0783ef651ef24f40f81f56597382a21188 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 22:51:12 +0800 Subject: [PATCH 04/18] test(interaction): focus attended regression coverage on complete attempts --- .../robots/dual_franka/test_exploration.py | 302 ++++-------------- .../rpent/cli/test_tui_contracts.py | 111 ------- .../rpent/planner/test_claude_contracts.py | 38 --- .../rpent/tools/test_human_in_the_loop.py | 56 ---- 4 files changed, 68 insertions(+), 439 deletions(-) delete mode 100644 tests/unit_tests/rpent/cli/test_tui_contracts.py diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index 46f6e6b84..73f4d8623 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -810,14 +810,12 @@ def run( return run -@pytest.mark.parametrize("last_verdict", ["failure", "abort", "unfinished"]) +@pytest.mark.parametrize("last_verdict", ["failure", "abort"]) def test_success_memory_survives_a_later_unsuccessful_session( - tmp_path, monkeypatch, last_verdict, run_interactive_cli + tmp_path, last_verdict, run_interactive_cli ): env = FakeEnv() - handlers = {} - calls = [] - waiting_tools = [] + handlers, calls = {}, [] def reader(input_queue, **kwargs): handlers["line"] = kwargs["line_handler"] @@ -825,49 +823,24 @@ def reader(input_queue, **kwargs): class Planner: def solve(self, *, toolkit, input_queue, **kwargs): - assert not toolkit._scene_ready - if calls: - assert (tmp_path / "memory/task-specific/dual_franka_t0.json").is_file() - verdict = "success" if not calls else last_verdict - calls.append(verdict) - if verdict == "unfinished": - import threading - - ready = threading.Event() - monkeypatch.setattr( - "builtins.print", lambda *args, **kwargs: ready.set() - ) - worker = threading.Thread( - target=lambda: call( - toolkit, "request_scene_reset", reason="pending" - ) - ) - waiting_tools.append(worker) - worker.start() - assert ready.wait(2) - assert handlers["line"]("/abort") - return PlannerResult() + outcome = "success" if not calls else last_verdict + calls.append(outcome) toolkit._operator_input = lambda *args: "done" call(toolkit, "request_scene_reset", reason="test") call(toolkit, "move_delta", arm="right", delta_xyz=[0.01, 0, 0]) - assert handlers["line"]("/" + verdict) + assert handlers["line"]("/" + outcome) assert input_queue.get(timeout=1) is None return PlannerResult() - sessions = 50 if last_verdict == "unfinished" else 2 - assert run_interactive_cli(Planner(), env, reader, sessions=sessions) == 0 - assert calls == ["success", last_verdict] + assert run_interactive_cli(Planner(), env, reader, sessions=2) == 0 published = json.loads( (tmp_path / "memory/task-specific/dual_franka_t0.json").read_text() ) + assert calls == ["success", last_verdict] and env.resets == 2 assert "session_001" in published["state_trace"] assert not ( tmp_path / "run/sessions/session_002/dual_franka_t0_recipe.jsonl" ).exists() - assert env.resets == (1 if last_verdict == "unfinished" else 2) - for worker in waiting_tools: - worker.join(2) - assert not worker.is_alive() @pytest.mark.parametrize("kind", ["reset", "verdict"]) @@ -901,36 +874,38 @@ def test_close_retires_waiting_request_without_robot_motion(setup, monkeypatch, assert broker.pending_kind is None -@pytest.mark.parametrize("pending_reset", [False, True]) -def test_attended_reply_resumes_same_toolkit_without_automatic_reset( - setup, monkeypatch, tmp_path, pending_reset +@pytest.mark.parametrize("kind", ["reset", "verdict"]) +def test_attended_confirmation_resumes_same_attempt_over_http( + setup, tmp_path, monkeypatch, kind ): + import asyncio import queue import threading + import httpx + from mcp import ClientSession + from mcp.client.streamable_http import streamable_http_client + from rpent.cli.attended import _solve_attended from rpent.tools.human_in_the_loop import HumanInTheLoopInput - t, env, replies = setup - if not pending_reset: - reset(t, replies) + toolkit, env, replies = setup + if kind == "verdict": + reset(toolkit, replies) broker = HumanInTheLoopInput(interactive=True) - t._operator_input = broker - assert not broker.route_line("Keep the completed task stages") + toolkit._operator_input = broker + ready, prompted = threading.Event(), threading.Event() + monkeypatch.setattr("builtins.print", lambda *a, **kw: prompted.set()) + monkeypatch.setattr( + "rpent.cli.attended.logger.info", + lambda message, *args: ( + ready.set() if "current operator request" in message else None + ), + ) inputs = queue.Queue() - first_reply = threading.Event() - second_reply = threading.Event() - prompt_ready = threading.Event() - monkeypatch.setattr("builtins.print", lambda *a, **kw: prompt_ready.set()) - workers = [] - calls = [] - servers = [] - - async def reset_over_http(url): - import httpx - from mcp import ClientSession - from mcp.client.streamable_http import streamable_http_client + calls, requests = [], [] + async def request(url): async with httpx.AsyncClient(trust_env=False) as client: async with streamable_http_client(url, http_client=client) as ( read, @@ -939,39 +914,29 @@ async def reset_over_http(url): ): async with ClientSession(read, write) as session: await session.initialize() - result = await session.call_tool( - "request_scene_reset", {"reason": "test"} + name = ( + "request_scene_reset" + if kind == "reset" + else "request_operator_verdict" ) + args = {"reason": "test"} if kind == "reset" else {} + result = await session.call_tool(name, args) assert not result.isError class Planner: - def solve(self, *, toolkit, user_message, **kwargs): - assert toolkit is t - server = kwargs["mcp_server"] - servers.append(server) - assert server._thread.is_alive() and not server._server.should_exit - calls.append(user_message) + def solve(self, *, toolkit, user_message, mcp_server, **kwargs): + calls.append(mcp_server) if len(calls) == 1: - if pending_reset: - import asyncio - - worker = threading.Thread( - target=lambda: asyncio.run(reset_over_http(server.url)) - ) - worker.start() - workers.append(worker) - assert prompt_ready.wait(2) - first_reply.set() - return PlannerResult( - messages=[{"role": "assistant", "content": "Please confirm"}] + worker = threading.Thread( + target=lambda: asyncio.run(request(mcp_server.url)) ) + requests.append(worker) + worker.start() + assert prompted.wait(2) + return PlannerResult() assert "SAME attempt" in user_message - assert user_message.count("Operator feedback, chronological") == 1 - assert "Keep the completed task stages" in user_message - assert t._scene_ready assert env.resets == 1 - t.request_direct_verdict("success", "test complete") - second_reply.set() + toolkit.request_direct_verdict("success", "checked") return PlannerResult() results = [] @@ -982,135 +947,36 @@ def solve(self, *, toolkit, user_message, **kwargs): operator_input=broker, state_output_dir=tmp_path / "session", keep_mcp_alive=True, - toolkit=t, + toolkit=toolkit, input_queue=inputs, system_prompt="test", - user_message="original task" + broker.feedback_prompt(), + user_message="test task", max_turns=10, ) ) ) runner.start() - assert first_reply.wait(2) - assert not second_reply.wait(0.15) - if pending_reset: - assert broker.route_line("/done 保留前半段成果") - else: - inputs.put("场景已调整，请继续后半段") - runner.join(3) - for worker in workers: - worker.join(2) - assert not runner.is_alive() and len(results) == 1 - assert len(calls) == 2 and env.resets == 1 - assert len(list((tmp_path / "session").glob("dialogue_*.json"))) == 2 - assert not env.moves - assert servers[0] is servers[1] - assert servers[0]._thread is None - - -@pytest.mark.parametrize("input_closed", [False, True]) -def test_attended_wait_eof_aborts_without_another_planner_call( - setup, tmp_path, monkeypatch, input_closed -): - import queue - - from rpent.cli.attended import _solve_attended - from rpent.tools.human_in_the_loop import HumanInTheLoopInput - - t, env, _ = setup - inputs = queue.Queue() - inputs.put(None) - calls = [] - broker = HumanInTheLoopInput(interactive=True) - if input_closed: - broker.close() - monkeypatch.setattr( - t, - "wait_active", - lambda: pytest.fail("closed input must abort before waiting"), + try: + assert ready.wait(2) + assert env.resets == int(kind == "verdict") + assert broker.route_line( + "/done" if kind == "reset" else "/continue keep progress" ) - - class Planner: - def solve(self, **kwargs): - calls.append(True) - return PlannerResult() - - _solve_attended( - Planner(), - operator_input=broker, - state_output_dir=tmp_path / "session", - toolkit=t, - input_queue=inputs, - user_message="task", - system_prompt="", - max_turns=10, - ) - assert calls == [True] - assert t.finalize_direct_verdict()["operator_aborted"] - assert not env.resets and not env.moves - - -@pytest.mark.parametrize("reply_style", ["shortcut", "request_id"]) -def test_attended_consumes_continue_once_without_duplicate_wait( - setup, tmp_path, monkeypatch, reply_style -): - import queue - import threading - - from rpent.cli.attended import _solve_attended - from rpent.tools.human_in_the_loop import HumanInTheLoopInput - - t, env, _ = setup - broker = HumanInTheLoopInput(interactive=True) - ready = threading.Event() - monkeypatch.setattr("builtins.print", lambda *a, **kw: ready.set()) - calls = [] - inputs = queue.Queue() - inputs.put(None) - - class Planner: - def solve(self, **kwargs): - calls.append(kwargs["user_message"]) - if len(calls) == 1: - replies = [] - thread = threading.Thread( - target=lambda: replies.append( - broker.request("continue?", lambda: None, kind="verdict") - ) - ) - thread.start() - assert ready.wait(2) - command = ( - "/continue" - if reply_style == "shortcut" - else f"/operator {broker._pending[0]} continue" - ) - assert broker.route_line(command) - thread.join(2) - assert replies == ["continue"] - return PlannerResult() - - _solve_attended( - Planner(), - operator_input=broker, - state_output_dir=tmp_path / "session", - toolkit=t, - input_queue=inputs, - user_message="task", - system_prompt="", - max_turns=10, - ) - assert len(calls) == 2 - assert "already answered continue" in calls[1] - assert broker.continue_revision == 1 - assert t.finalize_direct_verdict()["operator_aborted"] - assert not env.resets and not env.moves + runner.join(5) + for worker in requests: + worker.join(2) + assert not worker.is_alive() + assert not runner.is_alive() and results[0].error is None + assert len(calls) == 2 and calls[0] is calls[1] + assert calls[0]._thread is None + assert env.resets == 1 and not env.moves + finally: + inputs.put(None) + broker.close() + runner.join(5) -@pytest.mark.parametrize("phase", ["active", "reply_returned", "operator_request"]) -def test_attended_verdict_preserves_next_attempt_input( - setup, tmp_path, monkeypatch, phase -): +def test_attended_verdict_preserves_next_attempt_input(setup, tmp_path, monkeypatch): import queue import threading @@ -1142,30 +1008,14 @@ def verdict(outcome, notes): return True broker.bind_verdict(verdict) - request = None - if phase == "operator_request": - request_ready = threading.Event() - monkeypatch.setattr("builtins.print", lambda *a, **kw: request_ready.set()) - request = threading.Thread( - target=lambda: broker.request("Evaluate?", lambda: None, kind="verdict") - ) - request.start() - assert request_ready.wait(3) - - def info(message, *args): - if message.startswith("Waiting for"): - ready.set() - - monkeypatch.setattr(attended.logger, "info", info) def interrupt(turn): turn.interrupt_calls += 1 interrupted.set() def stream(turn): - if phase == "active": - ready.set() - assert interrupted.wait(3) + ready.set() + assert interrupted.wait(3) yield from turn.events monkeypatch.setattr(FakeTurn, "interrupt", interrupt) @@ -1194,17 +1044,12 @@ def stream(turn): inputs.put("Focus on the later stages") runner.join(5) assert not runner.is_alive() and results[0].error is None - assert FakeCodex.instances[0].thread.fake_turn.interrupt_calls == int( - phase == "active" - ) + assert FakeCodex.instances[0].thread.fake_turn.interrupt_calls == 1 finally: scope.cancel() broker.close() interrupted.set() runner.join(5) - if request is not None: - request.join(3) - assert not request.is_alive() delivered = threading.Event() @@ -1245,8 +1090,6 @@ def test_continue_captures_scene_after_operator_adjustment(setup, monkeypatch): original = env.get_observation scene = [10] reads = [] - published = [] - monkeypatch.setattr(t, "_publish_step", published.append) def observe(): reads.append(scene[0]) @@ -1269,14 +1112,5 @@ def operator(*args): assert observed.images == [ t.state.load_bytes("d455.png", step=result["observation_step"]) ] - assert result["observation_step"] == result["evidence_step"] + 1 assert reads == [10, 90] - assert [record.step_idx for record in published] == [ - result["evidence_step"], - result["observation_step"], - ] - assert [record.command for record in published] == [ - {"action": "observe_for_verdict"}, - {"action": "observe_current"}, - ] assert env.resets == 1 and not env.moves diff --git a/tests/unit_tests/rpent/cli/test_tui_contracts.py b/tests/unit_tests/rpent/cli/test_tui_contracts.py deleted file mode 100644 index 481d83aae..000000000 --- a/tests/unit_tests/rpent/cli/test_tui_contracts.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2026 The RPent Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import queue -import threading - -import pytest - -from rpent.cli.tui import initial_user_message, next_user_line -from rpent.session.input import SessionInputQueue - - -@pytest.mark.parametrize("read_line", [initial_user_message, next_user_line]) -@pytest.mark.parametrize("end_of_input", [None, "/quit", "/exit", "/q"]) -def test_scoped_input_preserves_feedback_and_real_exit(read_line, end_of_input): - source = queue.Queue() - scope = SessionInputQueue(source) - scope.put(" ") - scope.put(" inspect the other bowl ") - scope.put(end_of_input) - - assert read_line(scope) == "inspect the other bowl" - assert read_line(scope) is None - assert source.empty() - - -def test_cancelled_scope_leaves_feedback_and_real_eof_for_next_scope(): - source = queue.Queue() - previous = SessionInputQueue(source) - previous.cancel() - source.put("next attempt feedback") - source.put(None) - - assert previous.cancelled - assert previous.get(block=False) is None - assert previous.get(timeout=0) is None - current = SessionInputQueue(source) - assert not current.cancelled - assert current.get(block=False) == "next attempt feedback" - assert current.get(block=False) is None - assert source.empty() - - -def test_nested_scope_cleanup_does_not_cancel_attended_attempt(): - source = queue.Queue() - attempt = SessionInputQueue(source) - previous_reply = SessionInputQueue(attempt) - previous_reply.cancel() - attempt.put("continue after that reply") - - assert previous_reply.get() is None - assert not attempt.cancelled - current_reply = SessionInputQueue(attempt) - assert next_user_line(current_reply) == "continue after that reply" - attempt.cancel() - assert current_reply.get() is None - assert source.empty() - - -def test_scoped_input_retains_queue_nonblocking_and_timeout_contract(): - scope = SessionInputQueue(queue.Queue()) - with pytest.raises(queue.Empty): - scope.get(block=False) - with pytest.raises(queue.Empty): - scope.get(timeout=0) - with pytest.raises(queue.Empty): - scope.get(timeout=0.01) - with pytest.raises(ValueError, match="non-negative"): - scope.get(timeout=-1) - scope.put("available immediately") - assert scope.get(timeout=0) == "available immediately" - - -def test_cancel_retires_waiting_reader_without_consuming_later_input(): - reading = threading.Event() - - class ObservedQueue(queue.Queue): - def get(self, block=True, timeout=None): - reading.set() - return super().get(block=block, timeout=timeout) - - source = ObservedQueue() - scope = SessionInputQueue(source) - received = [] - reader = threading.Thread(target=lambda: received.append(scope.get()), daemon=True) - reader.start() - try: - assert reading.wait(1) - scope.cancel() - source.put("new attempt") - reader.join(timeout=1) - assert not reader.is_alive() - assert received == [None] - assert source.get(block=False) == "new attempt" - assert source.empty() - finally: - scope.cancel() - reader.join(timeout=1) diff --git a/tests/unit_tests/rpent/planner/test_claude_contracts.py b/tests/unit_tests/rpent/planner/test_claude_contracts.py index 657be3165..30dc59d56 100644 --- a/tests/unit_tests/rpent/planner/test_claude_contracts.py +++ b/tests/unit_tests/rpent/planner/test_claude_contracts.py @@ -594,44 +594,6 @@ def tool_template_output_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> monkeypatch.setattr("rpent.utils.templates.get_output_dir", lambda: tmp_path) -def test_terminal_adapter_close_does_not_stop_the_next_solve() -> None: - source = queue.Queue() - emitted = [] - queries = [] - interrupts = [] - - class Driver: - async def interrupt(self): - interrupts.append(True) - - async def query(self, text): - queries.append(text) - - async def run_two_solvers(): - first = _TerminalSessionAdapter( - toolkit=FakeToolkit(), input_queue=source, emit_user=emitted.append - ) - source.put(None) - await first.run(Driver()) - await first.close() - assert source.empty() - - second = _TerminalSessionAdapter( - toolkit=FakeToolkit(), input_queue=source, emit_user=emitted.append - ) - source.put("inspect the next attempt") - source.put("/quit") - await second.run(Driver()) - await second.close() - - asyncio.run(run_two_solvers()) - - assert emitted == ["inspect the next attempt"] - assert queries == emitted - assert len(interrupts) == 3 - assert source.empty() - - def test_terminal_adapter_cleanup_retires_delayed_reader( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit_tests/rpent/tools/test_human_in_the_loop.py b/tests/unit_tests/rpent/tools/test_human_in_the_loop.py index bec717788..cfe5fbcbe 100644 --- a/tests/unit_tests/rpent/tools/test_human_in_the_loop.py +++ b/tests/unit_tests/rpent/tools/test_human_in_the_loop.py @@ -20,42 +20,10 @@ from rpent.tools.human_in_the_loop import ( HumanInTheLoopInput, - OperatorCommand, - parse_operator_command, ) from rpent.tools.toolkit import ToolCancelled -@pytest.mark.parametrize( - "line,expected", - [ - ("please continue", None), - ("success", None), - ("/rpent-task --task-id 5", None), - (" /DONE scene restored ", OperatorCommand("done", "scene restored")), - ("/continue keep the bowl", OperatorCommand("continue", "keep the bowl")), - ("/succes", OperatorCommand("success")), - ("/failure missed grasp", OperatorCommand("failure", "missed grasp")), - ("/abort", OperatorCommand("abort")), - ( - "/operator request-1 continue keep the bowl", - OperatorCommand("operator", "keep the bowl", "request-1", "continue"), - ), - ], -) -def test_shared_operator_command_parser(line, expected): - assert parse_operator_command(line) == expected - - -@pytest.mark.parametrize("line", ["/operator", "/operator request-1"]) -def test_incomplete_operator_command_never_becomes_chat(line): - with pytest.raises(ValueError, match="Usage: /operator"): - parse_operator_command(line) - broker = HumanInTheLoopInput(interactive=True) - assert broker.route_line(line) - assert broker.feedback_prompt() == "" - - def test_operator_replies_are_request_scoped_and_do_not_consume_steering(monkeypatch): broker = HumanInTheLoopInput(interactive=True) ready = threading.Event() @@ -192,30 +160,6 @@ def test_success_is_control_and_never_steering(): assert len(calls) == 1 -@pytest.mark.parametrize("verdict", ["success", "failure", "abort"]) -def test_verdict_notes_are_control_not_steering(tmp_path, verdict): - broker = HumanInTheLoopInput( - interactive=True, feedback_path=tmp_path / "feedback.jsonl" - ) - calls = [] - broker.bind_verdict(lambda value, notes: calls.append((value, notes)) or True) - assert broker.route_line(f"/{verdict} 建议采用混合控制") - assert calls == [(verdict, "建议采用混合控制")] - assert "建议采用混合控制" in broker.feedback_prompt() - - -def test_failed_prompt_display_does_not_leave_pending_request(monkeypatch): - broker = HumanInTheLoopInput(interactive=True) - - def fail(*args, **kwargs): - raise OSError("terminal closed") - - monkeypatch.setattr("builtins.print", fail) - with pytest.raises(OSError, match="terminal closed"): - broker.request("reset", lambda: None, kind="reset") - assert broker.pending_kind is None - - def test_only_slash_verdicts_are_control_commands(): broker = HumanInTheLoopInput(interactive=True) verdicts = [] From e1068f83b6205705380b5b2a2a8c5b8589dbc5af Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 22:54:11 +0800 Subject: [PATCH 05/18] test(dashboard): narrow operator coverage to complete web task flows --- .../dual_franka/test_dashboard_operator.py | 128 ++++++------------ .../rpent/dashboard/test_server_contracts.py | 39 ++---- .../rpent/dashboard/test_session_contracts.py | 49 ++----- .../rpent/dashboard/test_state_contracts.py | 65 --------- 4 files changed, 60 insertions(+), 221 deletions(-) diff --git a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py index 2bde6997d..766fb7d23 100644 --- a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py +++ b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py @@ -100,7 +100,18 @@ def message(pending, text): def test_reset_waits_for_one_matching_reply(pending_reset, answer): toolkit, env, _, client, pending, future = pending_reset reply = message(pending, f"/{answer} scene checked") - assert env.resets == 0 + for context in ( + {"generation": -1}, + {**reply["operator_context"], "request_id": "stale"}, + ): + assert ( + client.post( + "/api/session/messages", + json={"text": "/done", "operator_context": context}, + ).status_code + == 422 + ) + assert not future.done() and env.resets == 0 assert client.post("/api/session/messages", json=reply).status_code == 202 result = future.result(3) assert env.resets == int(answer == "done") @@ -112,45 +123,22 @@ def test_reset_waits_for_one_matching_reply(pending_reset, answer): assert env.resets == int(answer == "done") -@pytest.mark.parametrize( - "context", - [ - {"generation": -1}, - {"generation": 1, "request_id": "stale"}, - [], - ], -) -def test_invalid_reply_cannot_release_reset(pending_reset, context): - _, env, _, client, pending, future = pending_reset - reply = {**message(pending, "/done"), "operator_context": context} - assert client.post("/api/session/messages", json=reply).status_code == 422 - assert not future.done() and env.resets == 0 - - -@pytest.mark.parametrize("verdict", ["success", "failure", "abort"]) -def test_verdict_closes_only_current_task(pending_reset, verdict): +def test_verdict_closes_only_current_task(pending_reset): toolkit, env, state, client, pending, future = pending_reset assert ( client.post("/api/session/messages", json=message(pending, "/done")).status_code == 202 ) future.result(3) - reply = message(pending, f"/{verdict} checked") + reply = message(pending, "/success checked") assert client.post("/api/session/messages", json=reply).status_code == 202 assert call(toolkit, "open_gripper", arm="right")["motion_refused"] toolkit.finalize_direct_verdict() - assert toolkit.solved() is (verdict == "success") + assert toolkit.solved() assert client.post("/api/session/messages", json=reply).status_code == 422 state.unbind_toolkit(toolkit) state.complete_task(state="cancelled") - assert ( - state._task_state - == { - "success": "succeeded", - "failure": "failed", - "abort": "cancelled", - }[verdict] - ) + assert client.get("/api/session/state").json()["state"] == "succeeded" state.request_task({"task_id": 4}) state.wait_for_task(0) state.bind_toolkit(toolkit) @@ -185,28 +173,9 @@ def test_cancel_releases_operator_wait_without_reset(pending_reset): assert env.resets == 0 and state.operator.snapshot() is None -@pytest.mark.parametrize( - "explore,verdict,auto_merge,error,replacement", - [ - (False, "success", True, None, False), - (True, "success", True, None, False), - (True, "success", False, None, False), - (True, "success", True, "planner failed", False), - (True, "failure", True, None, False), - (True, "abort", True, None, False), - (True, None, True, None, True), - (True, "success", True, None, True), - ], -) +@pytest.mark.parametrize("verdict", ["success", "failure", "abort"]) def test_task_uses_web_operator_and_publishes_only_confirmed_success( - tmp_path, - monkeypatch, - dual_franka_robot_config, - explore, - verdict, - auto_merge, - error, - replacement, + tmp_path, monkeypatch, dual_franka_robot_config, verdict ): from rpent.cli import dashboard, main @@ -217,24 +186,24 @@ def test_task_uses_web_operator_and_publishes_only_confirmed_success( "--robot", "dual_franka", "--dashboard", + "--explore", "--task-id", "3", - "--robot-config", - str(dual_franka_robot_config), "--planner", "codex", - "--memory-profile", - "local", + "--auto-merge-memory", + "--robot-config", + str(dual_franka_robot_config), "--memory-dir", str(tmp_path / "memory"), - "--auto-merge-memory" if auto_merge else "--no-auto-merge-memory", + "--output-dir", + str(tmp_path / "run"), ] ) - args.explore = explore env = FakeEnv() spec = replace( robot_spec.get_robot_spec(), - init_runtime=lambda *args: ( + init_runtime=lambda *a: ( [], {"env": env, "model": None, "task_description": "test"}, ), @@ -246,27 +215,18 @@ def test_task_uses_web_operator_and_publishes_only_confirmed_success( def solve(**kwargs): toolkit = kwargs["toolkit"] - assert kwargs["dashboard_interaction"] is state assert env.resets == 0 - assert "error" in call( - toolkit, "move_delta", arm="right", delta_xyz=[0, 0, 0.01] - ) with ThreadPoolExecutor() as pool: future = pool.submit(call, toolkit, "request_scene_reset", reason="test") - try: - wait_for_request(state.operator) - pending = state.operator_snapshot() - state.operator_reply( - pending["generation"], pending["pending"]["id"], "done" - ) - future.result(3) - finally: - state.operator.close() - if verdict is not None: - state.operator_finish(1, verdict, "checked") - if replacement: - state.request_task({"task_id": 4}) - return PlannerResult(error=error) + wait_for_request(state.operator) + pending = state.operator_snapshot() + state.operator_reply( + pending["generation"], pending["pending"]["id"], "done" + ) + future.result(3) + call(toolkit, "move_delta", arm="right", delta_xyz=[0, 0, 0.01]) + state.operator_finish(pending["generation"], verdict, "checked") + return PlannerResult() monkeypatch.setattr( dashboard, "build_planner", lambda *a, **kw: SimpleNamespace(solve=solve) @@ -281,25 +241,17 @@ def solve(**kwargs): unique_components={"env"}, session_root=tmp_path, ) - == error - ) - assert env.resets == 1 - published = ( - explore - and verdict == "success" - and auto_merge - and error is None - and not replacement + is None ) + assert env.resets == 1 and len(env.moves) == 1 for filename in ("dual_franka_t3.json", "dual_franka_t3_recipe.jsonl"): - assert (tmp_path / "memory/task-specific" / filename).is_file() is published - if explore and verdict in {"success", "failure"}: + assert (tmp_path / "memory/task-specific" / filename).is_file() is ( + verdict == "success" + ) + if verdict != "abort": evidence = list((tmp_path / "memory").rglob(f"*-{verdict}.json")) assert len(evidence) == 1 assert ( json.loads(evidence[0].read_text())["verdict"]["operator_verdict"] == verdict ) - state.complete_task(state="cancelled", error=error) - if replacement: - assert state.wait_for_task(0).request == {"task_id": 4} diff --git a/tests/unit_tests/rpent/dashboard/test_server_contracts.py b/tests/unit_tests/rpent/dashboard/test_server_contracts.py index b2e12c53d..5a54023f1 100644 --- a/tests/unit_tests/rpent/dashboard/test_server_contracts.py +++ b/tests/unit_tests/rpent/dashboard/test_server_contracts.py @@ -244,39 +244,18 @@ def test_healthz_still_reports_transport_liveness_only(state: DashboardState) -> assert resp.json() == {"ok": True} -@pytest.mark.parametrize( - "text", - [ - "/done", - "/continue keep the completed half", - "/success", - "/failure missed grasp", - "/abort", - "/operator", - "/operator previous-request done", - ], -) -@pytest.mark.parametrize( - "operator_context", - [None, {"generation": -1, "request_id": "previous-request"}], -) -def test_generic_messages_do_not_enable_operator_control(state, text, operator_context): +def test_generic_messages_do_not_enable_operator_control(state): state.shared_services_ready() state.request_task({"seed": 1}) state.wait_for_task(timeout=0) state.set_planner_activity("idle", accepting_input=True) client = _client(_server(state)) - - response = client.post( - "/api/session/messages", - json={"text": text, "operator_context": operator_context}, - ) - - assert response.status_code == 202 - assert state.claim_next_pending_message().text == text - assert state.claim_next_pending_message() is None - snapshot = client.get("/api/session/state").json() - assert snapshot["operator"]["enabled"] is False - assert snapshot["operator"]["verdict"] is None - assert snapshot["task_generation"] == 1 + for text in ("/done", "/success"): + response = client.post( + "/api/session/messages", + json={"text": text, "operator_context": {"generation": -1}}, + ) + assert response.status_code == 202 + assert state.claim_next_pending_message().text == text + assert client.get("/api/session/state").json()["operator"]["enabled"] is False assert not state.task_replacement_requested diff --git a/tests/unit_tests/rpent/dashboard/test_session_contracts.py b/tests/unit_tests/rpent/dashboard/test_session_contracts.py index ce2efc31f..2a53e6914 100644 --- a/tests/unit_tests/rpent/dashboard/test_session_contracts.py +++ b/tests/unit_tests/rpent/dashboard/test_session_contracts.py @@ -175,13 +175,11 @@ def test_dashboard_session_stops_shared_daemons_in_reverse_after_cleanup_error( @pytest.mark.parametrize("merge_fails", [False, True]) @pytest.mark.parametrize("sessions", [1, 2]) -@pytest.mark.parametrize("real_robot", [False, True]) def test_dashboard_exploration_finalizes_memory_and_reports_merge_failures( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, merge_fails: bool, sessions: int, - real_robot: bool, ) -> None: from rpent.cli import dashboard as dashboard_cli @@ -197,7 +195,6 @@ def merge_memory(self, **kwargs: Any) -> dict[str, int]: class FakeToolkit: memory = FakeMemoryManager() - direct_verdict_requested = False def cancel_active_and_wait(self) -> None: pass @@ -236,9 +233,6 @@ def unbind_toolkit(self, toolkit: Any) -> None: def report_task_warning(self, warning: str) -> None: self.warnings.append(warning) - def operator_changed(self) -> None: - pass - class FakePlanner: def solve(self, **kwargs: Any) -> PlannerResult: del kwargs @@ -260,13 +254,9 @@ def solve(self, **kwargs: Any) -> PlannerResult: finalize_run=None, prepare_memory=None, supports_exploration=True, - is_real_robot=real_robot, - supports_human_interactive_exploration=real_robot, + is_real_robot=False, parse_config=lambda args: run_config, - init_runtime=lambda *args: ( - [], - {"options": "current-task", "task_description": "current-task"}, - ), + init_runtime=lambda *args: ([], {}), prompts=PromptBundle( system=lambda variables: "system", user=lambda variables: "user", @@ -294,16 +284,9 @@ def solve(self, **kwargs: Any) -> PlannerResult: from rpent.cli import main as main_cli monkeypatch.setattr(main_cli, "get_robot_spec", lambda name: robot_spec) - - def get_toolkit(*args, **kwargs): - assert kwargs["runtime_kwargs"]["options"] == "shared-default" - assert kwargs["runtime_kwargs"]["task_description"] == "current-task" - assert kwargs["runtime_kwargs"]["shared_client"] == "client" - if real_robot: - assert kwargs["operator_input"] is state.operator - return FakeToolkit() - - monkeypatch.setattr(dashboard_cli, "get_toolkit", get_toolkit) + monkeypatch.setattr( + dashboard_cli, "get_toolkit", lambda *args, **kwargs: FakeToolkit() + ) monkeypatch.setattr( dashboard_cli, "build_planner", lambda *args, **kwargs: FakePlanner() ) @@ -313,7 +296,7 @@ def get_toolkit(*args, **kwargs): robot_spec=robot_spec, state=state, claimed=claimed, - shared_runtime_kwargs={"options": "shared-default", "shared_client": "client"}, + shared_runtime_kwargs={}, unique_components=set(), session_root=tmp_path / "session", ) @@ -382,21 +365,11 @@ def solve(**kwargs): ), finalize_run=finalized.append, ) - args = SimpleNamespace( - verbose=False, - robot_name="yam", - explore=False, - auto_merge_memory=False, - planner="api", - base_url=None, - model="offline", - max_tokens=128, - planner_timeout_s=10, - reasoning_effort=None, - claude_code_max_budget_usd=None, - no_images=False, - max_turns=2, - ) + from rpent.cli.main import _build_argparser + + args = _build_argparser().parse_args(["--robot", "yam", "--planner", "api"]) + args.robot_name = "yam" + args.auto_merge_memory = False state = DashboardState(output_dir=tmp_path, dashboard_spec=YAM_DASHBOARD_SPEC) monkeypatch.setattr(dashboard_cli, "get_toolkit", get_toolkit) monkeypatch.setattr( diff --git a/tests/unit_tests/rpent/dashboard/test_state_contracts.py b/tests/unit_tests/rpent/dashboard/test_state_contracts.py index 9e8c8f062..39de8f3d4 100644 --- a/tests/unit_tests/rpent/dashboard/test_state_contracts.py +++ b/tests/unit_tests/rpent/dashboard/test_state_contracts.py @@ -87,71 +87,6 @@ def _claim_started_task( return claimed -@pytest.mark.parametrize("command", ["/done", "/success", "/failure", "/abort"]) -@pytest.mark.parametrize("web_operator", [False, True]) -def test_external_robot_operator_commands_keep_terminal_routing( - tmp_path, command, web_operator -): - from robots.yam.robot_spec import YAM_DASHBOARD_SPEC - - state = DashboardState(output_dir=tmp_path, dashboard_spec=YAM_DASHBOARD_SPEC) - if web_operator: - state.operator = MagicMock() - with pytest.raises(ValueError, match="Use the operator terminal"): - state.submit_input(command, operator_context={"generation": -1}) - if web_operator: - state.operator.reply.assert_not_called() - state.operator.close.assert_not_called() - else: - assert state.operator is None - - -def test_external_robot_continue_uses_toolkit_without_web_operator(tmp_path): - from robots.yam.robot_spec import YAM_DASHBOARD_SPEC - - state = DashboardState( - output_dir=tmp_path, - dashboard_spec={ - **DASHBOARD_SPEC, - "operator_commands": YAM_DASHBOARD_SPEC["operator_commands"], - }, - ) - state.shared_services_ready() - _claim_started_task(state) - state.set_planner_activity("idle", accepting_input=True) - toolkit = MagicMock(spec=Toolkit) - toolkit.exploration_continuation.return_value = "check episode and continue" - state.bind_toolkit(toolkit) - - assert state.submit_input("/continue", operator_context={"generation": -1}) == { - "command": "/continue" - } - - toolkit.exploration_continuation.assert_called_once_with(explicit=True) - assert state.claim_next_pending_message().text == "check episode and continue" - assert state.operator is None - - -@pytest.mark.parametrize("web_operator", [False, True]) -def test_task_commands_ignore_operator_reply_context(tmp_path, web_operator): - state = _ready_state(tmp_path) - _claim_started_task(state) - if web_operator: - state.operator = MagicMock() - state.operator.snapshot.return_value = None - - state.submit_input( - "/rpent-task place 2", - operator_context={"generation": -1, "request_id": "previous-request"}, - ) - - assert state.task_replacement_requested - assert state.snapshot()["pending_task"] == {"label": "place / seed 2"} - if web_operator: - state.operator.reply.assert_not_called() - state.operator.close.assert_not_called() - - def test_dashboard_task_commands_validate_and_latest_unclaimed_request_wins( tmp_path: Path, ) -> None: From 93c4af4349bf0e887b91ef5d19416a1aa2e34da1 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 23:12:59 +0800 Subject: [PATCH 06/18] test(interaction): keep only primary attended execution paths --- .../robots/dual_franka/test_exploration.py | 426 +++--------------- .../rpent/cli/test_main_contracts.py | 41 +- .../rpent/evaluation/test_result.py | 28 -- .../rpent/planner/test_claude_contracts.py | 50 -- .../rpent/planner/test_codex_contracts.py | 76 +--- 5 files changed, 73 insertions(+), 548 deletions(-) diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index 73f4d8623..7c83c4e69 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -173,22 +173,10 @@ def fail(): assert not env.moves -@pytest.mark.parametrize( - "response", - [ - "abort", - "abort calibration board still attached", - " ABORT operator stop ", - None, - ], -) -def test_operator_abort_allows_finish_even_with_budget_and_never_succeeds( - setup, response -): +def test_operator_abort_allows_finish_even_with_budget_and_never_succeeds(setup): t, env, replies = setup - replies.append(response) - reset_result = call(t, "request_scene_reset", reason="initial") - assert reset_result["operator_aborted"] + replies.append("abort") + call(t, "request_scene_reset", reason="initial") result = call(t, "finish", status="success", summary="stop") assert result["_finish"] and result["operator_aborted"] assert result["status"] == "failure" and not t.solved() and env.resets == 0 @@ -583,96 +571,90 @@ def test_direct_success_with_failed_observation_does_not_publish(setup): assert t.write_recipe("dual_franka_t0") is None -@pytest.mark.parametrize( - "verdict,auto_merge_memory,planner_error,session_count,consume_cancel", - [ - ("success", True, None, 2, True), - ("success", False, None, 1, False), - ("success", True, "planner transport failed", 1, True), - ("failure", True, None, 2, False), - ("failure", True, "planner transport failed", 2, True), - ("abort", True, None, 1, False), - ("abort", True, "planner transport failed", 1, True), - ], -) +@pytest.mark.parametrize("robot_name", ["dual_franka", "libero"]) +@pytest.mark.parametrize("verdict", ["success", "failure", "abort"]) +@pytest.mark.parametrize("planner_error", [None, "planner transport failed"]) def test_cli_direct_verdict_finalizes_and_merges_only_without_errors( tmp_path, - run_interactive_cli, + monkeypatch, + dual_franka_robot_config, verdict, planner_error, - session_count, - consume_cancel, - auto_merge_memory, + robot_name, ): + import sys + from dataclasses import replace + from types import SimpleNamespace + + from rpent.cli import main as cli + env = FakeEnv() handlers = {} - solve_calls = [] def reader(input_queue, **kwargs): handlers["line"] = kwargs["line_handler"] - handlers["inputs"] = input_queue input_queue.put("test task") + monkeypatch.setattr(cli, "start_interactive_reader", reader) + class Planner: def solve(self, *, toolkit, input_queue, **kwargs): - import queue - - with pytest.raises(queue.Empty): - input_queue.get(block=False) - if solve_calls: - assert "write a grounded outcome summary" in kwargs["user_message"] - assert "Focus on the later task stages" in kwargs["user_message"] - assert not toolkit._scene_ready - if verdict == "success" and auto_merge_memory: - assert ( - tmp_path / "memory/task-specific/dual_franka_t0_recipe.jsonl" - ).is_file() - solve_calls.append(True) - assert not handlers["line"]("Focus on the later task stages") toolkit._operator_input = lambda *args: "done" call(toolkit, "request_scene_reset", reason="test") call(toolkit, "move_delta", arm="right", delta_xyz=[0.01, 0, 0]) - assert handlers["line"]("/" + verdict + " 建议采用混合控制") - assert handlers["line"]("/" + verdict + " 重复提交") - if consume_cancel: - assert input_queue.get(timeout=1) is None - assert handlers["inputs"].empty() - return PlannerResult(error=planner_error) - - result = run_interactive_cli( - Planner(), - env, - reader, - sessions=session_count, - auto_merge_memory=auto_merge_memory, + assert handlers["line"]("/" + verdict) + assert input_queue.get(timeout=1) is None + return SimpleNamespace( + finish_result=None, messages=[], stats={}, error=planner_error + ) + + spec = replace( + robot_spec.get_robot_spec(), + init_runtime=lambda *args: ( + [], + { + "env": env, + "model": None, + "task_description": "test", + }, + ), ) - assert result == (1 if planner_error and verdict != "failure" else 0) - if verdict == "failure": - failures = list((tmp_path / "memory/_internal/inbox").rglob("*-failure.json")) - assert len(failures) == session_count - assert json.loads(failures[0].read_text())["verdict"]["status"] == "failure" - assert ( - json.loads(failures[0].read_text())["verdict"]["operator_notes"] - == "建议采用混合控制" - ) - assert len(solve_calls) == ( - session_count - if verdict == "failure" or (verdict == "success" and not planner_error) - else 1 + monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) + original_toolkit_factory = cli.get_toolkit + monkeypatch.setattr( + cli, + "get_toolkit", + lambda name, **kwargs: original_toolkit_factory("dual_franka", **kwargs), ) - if verdict == "success" and not planner_error: - assert ( - len(list((tmp_path / "run/sessions").glob("*/dual_franka_t0_recipe.jsonl"))) - == session_count - ) + monkeypatch.setattr(cli, "build_planner", lambda *args, **kwargs: Planner()) + monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True)) + monkeypatch.setattr( + sys, + "argv", + [ + "rpent", + "--robot", + robot_name, + "--planner", + "codex", + "--explore", + "--interactive", + "--auto-merge-memory", + "--output-dir", + str(tmp_path / "run"), + "--memory-dir", + str(tmp_path / "memory"), + "--robot-config", + str(dual_franka_robot_config), + ], + ) + assert cli.main() == (1 if planner_error and verdict != "failure" else 0) assert (tmp_path / "memory/task-specific/dual_franka_t0.json").is_file() == ( - auto_merge_memory and verdict == "success" and planner_error is None + verdict == "success" and planner_error is None ) assert ( tmp_path / "memory/task-specific/dual_franka_t0_recipe.jsonl" - ).is_file() == ( - auto_merge_memory and verdict == "success" and planner_error is None - ) + ).is_file() == (verdict == "success" and planner_error is None) events = json.loads( (tmp_path / "run/sessions/session_001/operator_events.json").read_text() ) @@ -753,130 +735,8 @@ def observe(): np.testing.assert_allclose(env.moves[0][1], [0.01, 0, 0]) -@pytest.fixture -def run_interactive_cli(tmp_path, monkeypatch, dual_franka_robot_config): - import sys - from dataclasses import replace - from types import SimpleNamespace - - from rpent.cli import main as cli - - monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True)) - - def run( - planner, - env, - reader, - *, - sessions, - auto_merge_memory=True, - ): - spec = replace( - robot_spec.get_robot_spec(), - init_runtime=lambda *a: ( - [], - {"env": env, "model": None, "task_description": "test"}, - ), - ) - monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) - monkeypatch.setattr(cli, "start_interactive_reader", reader) - monkeypatch.setattr(cli, "build_planner", lambda *a, **kw: planner) - monkeypatch.setattr( - sys, - "argv", - [ - "rpent", - "--robot", - "dual_franka", - "--planner", - "codex", - "--explore", - "--interactive", - "--explore-sessions", - str(sessions), - "--output-dir", - str(tmp_path / "run"), - "--memory-dir", - str(tmp_path / "memory"), - "--robot-config", - str(dual_franka_robot_config), - "--auto-merge-memory" - if auto_merge_memory - else "--no-auto-merge-memory", - ], - ) - return cli.main() - - return run - - -@pytest.mark.parametrize("last_verdict", ["failure", "abort"]) -def test_success_memory_survives_a_later_unsuccessful_session( - tmp_path, last_verdict, run_interactive_cli -): - env = FakeEnv() - handlers, calls = {}, [] - - def reader(input_queue, **kwargs): - handlers["line"] = kwargs["line_handler"] - input_queue.put("test task") - - class Planner: - def solve(self, *, toolkit, input_queue, **kwargs): - outcome = "success" if not calls else last_verdict - calls.append(outcome) - toolkit._operator_input = lambda *args: "done" - call(toolkit, "request_scene_reset", reason="test") - call(toolkit, "move_delta", arm="right", delta_xyz=[0.01, 0, 0]) - assert handlers["line"]("/" + outcome) - assert input_queue.get(timeout=1) is None - return PlannerResult() - - assert run_interactive_cli(Planner(), env, reader, sessions=2) == 0 - published = json.loads( - (tmp_path / "memory/task-specific/dual_franka_t0.json").read_text() - ) - assert calls == ["success", last_verdict] and env.resets == 2 - assert "session_001" in published["state_trace"] - assert not ( - tmp_path / "run/sessions/session_002/dual_franka_t0_recipe.jsonl" - ).exists() - - -@pytest.mark.parametrize("kind", ["reset", "verdict"]) -def test_close_retires_waiting_request_without_robot_motion(setup, monkeypatch, kind): - import threading - - from rpent.tools.human_in_the_loop import HumanInTheLoopInput - - t, env, replies = setup - if kind == "verdict": - reset(t, replies) - resets_before = env.resets - broker = HumanInTheLoopInput(interactive=True) - t._operator_input = broker - ready = threading.Event() - monkeypatch.setattr("builtins.print", lambda *args, **kwargs: ready.set()) - result = [] - tool = "request_scene_reset" if kind == "reset" else "request_operator_verdict" - kwargs = {"reason": "test cancellation"} if kind == "reset" else {} - worker = threading.Thread(target=lambda: result.append(call(t, tool, **kwargs))) - worker.start() - assert ready.wait(2) - old_id = broker._pending[0] - t.close() - worker.join(2) - assert not worker.is_alive() - assert broker.pending_kind is None - assert env.resets == resets_before and not env.moves - assert call(t, "request_scene_reset", reason="late call")["motion_refused"] - assert broker.route_line(f"/operator {old_id} done") - assert broker.pending_kind is None - - -@pytest.mark.parametrize("kind", ["reset", "verdict"]) def test_attended_confirmation_resumes_same_attempt_over_http( - setup, tmp_path, monkeypatch, kind + setup, tmp_path, monkeypatch ): import asyncio import queue @@ -890,8 +750,6 @@ def test_attended_confirmation_resumes_same_attempt_over_http( from rpent.tools.human_in_the_loop import HumanInTheLoopInput toolkit, env, replies = setup - if kind == "verdict": - reset(toolkit, replies) broker = HumanInTheLoopInput(interactive=True) toolkit._operator_input = broker ready, prompted = threading.Event(), threading.Event() @@ -914,12 +772,8 @@ async def request(url): ): async with ClientSession(read, write) as session: await session.initialize() - name = ( - "request_scene_reset" - if kind == "reset" - else "request_operator_verdict" - ) - args = {"reason": "test"} if kind == "reset" else {} + name = "request_scene_reset" + args = {"reason": "test"} result = await session.call_tool(name, args) assert not result.isError @@ -958,10 +812,8 @@ def solve(self, *, toolkit, user_message, mcp_server, **kwargs): runner.start() try: assert ready.wait(2) - assert env.resets == int(kind == "verdict") - assert broker.route_line( - "/done" if kind == "reset" else "/continue keep progress" - ) + assert env.resets == 0 + assert broker.route_line("/done") runner.join(5) for worker in requests: worker.join(2) @@ -974,143 +826,3 @@ def solve(self, *, toolkit, user_message, mcp_server, **kwargs): inputs.put(None) broker.close() runner.join(5) - - -def test_attended_verdict_preserves_next_attempt_input(setup, tmp_path, monkeypatch): - import queue - import threading - - from rpent.cli import attended - from rpent.planner.utils.http_mcp_server import HttpMcpServer - from rpent.session.input import SessionInputQueue - from rpent.tools.human_in_the_loop import HumanInTheLoopInput - from tests.unit_tests.rpent.planner.test_codex_contracts import ( - FakeCodex, - FakeTurn, - RecordingSink, - install_fake_backend, - make_planner, - ) - - install_fake_backend(monkeypatch) - monkeypatch.setattr("rpent.planner.codex.HttpMcpServer", HttpMcpServer) - toolkit, env, replies = setup - reset(toolkit, replies) - inputs = queue.Queue() - scope = SessionInputQueue(inputs) - broker = HumanInTheLoopInput(interactive=True) - ready, interrupted = threading.Event(), threading.Event() - - def verdict(outcome, notes): - assert toolkit.request_direct_verdict(outcome, notes) - scope.cancel() - broker.cancel_pending() - return True - - broker.bind_verdict(verdict) - - def interrupt(turn): - turn.interrupt_calls += 1 - interrupted.set() - - def stream(turn): - ready.set() - assert interrupted.wait(3) - yield from turn.events - - monkeypatch.setattr(FakeTurn, "interrupt", interrupt) - monkeypatch.setattr(FakeTurn, "stream", stream) - planner = make_planner(tmp_path, RecordingSink(), timeout_s=5) - results = [] - runner = threading.Thread( - target=lambda: results.append( - attended._solve_attended( - planner, - operator_input=broker, - state_output_dir=tmp_path / "attempt1", - keep_mcp_alive=True, - toolkit=toolkit, - input_queue=scope, - system_prompt="test", - user_message="task", - max_turns=10, - ) - ) - ) - runner.start() - try: - assert ready.wait(3) - assert broker.route_line("/success") - inputs.put("Focus on the later stages") - runner.join(5) - assert not runner.is_alive() and results[0].error is None - assert FakeCodex.instances[0].thread.fake_turn.interrupt_calls == 1 - finally: - scope.cancel() - broker.close() - interrupted.set() - runner.join(5) - - delivered = threading.Event() - - def steer(turn, text): - turn.steered.append(text) - delivered.set() - - def next_stream(turn): - assert delivered.wait(3) - yield from turn.events - - monkeypatch.setattr(FakeTurn, "steer", steer) - monkeypatch.setattr(FakeTurn, "stream", next_stream) - next_toolkit = DualFrankaToolkit( - runtime_kwargs={"env": env, "model": None, "task_description": "test"}, - dashboard_events=NullDashboardEventSink(), - memory=toolkit.memory, - mode="exploration", - state_output_dir=tmp_path / "attempt2", - operator_input=broker, - ) - result = planner.solve( - system_prompt="test", - user_message="next attempt", - toolkit=next_toolkit, - max_turns=10, - input_queue=SessionInputQueue(inputs), - ) - assert result.error is None - turn = FakeCodex.instances[-1].thread.fake_turn - assert turn.steered == ["Focus on the later stages"] and turn.interrupt_calls == 0 - assert inputs.empty() - - -def test_continue_captures_scene_after_operator_adjustment(setup, monkeypatch): - t, env, replies = setup - reset(t, replies) - original = env.get_observation - scene = [10] - reads = [] - - def observe(): - reads.append(scene[0]) - obs = original() - obs["d455_images"][:] = scene[0] - return obs - - env.get_observation = observe - - def operator(*args): - scene[0] = 90 - return "continue moved object" - - t._operator_input = operator - observed = t.execute_tool("request_operator_verdict", {}) - result = observed.data - assert result["status"] == "continue" - assert np.all(t.state.load("d455.png", step=result["evidence_step"]) == 10) - assert np.all(t.state.load("d455.png", step=result["observation_step"]) == 90) - assert observed.images == [ - t.state.load_bytes("d455.png", step=result["observation_step"]) - ] - assert reads == [10, 90] - assert env.resets == 1 and not env.moves diff --git a/tests/unit_tests/rpent/cli/test_main_contracts.py b/tests/unit_tests/rpent/cli/test_main_contracts.py index 16702e92c..9f9be6d60 100644 --- a/tests/unit_tests/rpent/cli/test_main_contracts.py +++ b/tests/unit_tests/rpent/cli/test_main_contracts.py @@ -111,9 +111,6 @@ def test_public_cli_help_runs_from_source_checkout_without_a_robot_runtime() -> "--interactive", ): assert option in result.stdout - assert "--dashboard-language" in result.stdout - assert "--dashboard-record" not in result.stdout - assert "--dashboard-camera-stream" not in result.stdout @pytest.mark.parametrize( @@ -188,21 +185,6 @@ def test_robot_and_env_aliases_are_mutually_exclusive( @pytest.mark.parametrize( ("argv", "message"), [ - ( - ["--robot", "libero", "--planner", "api", "--reasoning-effort", "ultra"], - "requires --planner codex", - ), - ( - [ - "--robot", - "libero", - "--planner", - "claude_code", - "--reasoning-effort", - "ultra", - ], - "requires --planner codex", - ), ( ["--robot", "libero", "--dashboard", "--interactive"], "cannot be used together", @@ -373,8 +355,8 @@ def test_real_robot_terminal_requirement_fails_before_runtime( cli = _cli_module() spec = SimpleNamespace( is_real_robot=True, - dashboard=None, supports_human_interactive_exploration=False, + dashboard=None, add_cli_args=lambda parser, use_dashboard: None, ) monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) @@ -803,24 +785,3 @@ def finalize_run(context: RunFinalizationContext) -> Path: } assert robot_toolkit.closed is True assert daemon.stopped is True - - -def test_codex_ultra_effort_reaches_robot_config_parser( - monkeypatch: pytest.MonkeyPatch, -) -> None: - _, args = _capture_validated_args( - monkeypatch, - [ - "--robot", - "libero", - "--planner", - "codex", - "--model", - "gpt-6-astra", - "--reasoning-effort", - "ultra", - ], - ) - - assert args.model == "gpt-6-astra" - assert args.reasoning_effort == "ultra" diff --git a/tests/unit_tests/rpent/evaluation/test_result.py b/tests/unit_tests/rpent/evaluation/test_result.py index 8d39b20d7..f43b031fa 100644 --- a/tests/unit_tests/rpent/evaluation/test_result.py +++ b/tests/unit_tests/rpent/evaluation/test_result.py @@ -21,34 +21,6 @@ import pytest from rpent.evaluation import write_json_atomic -from rpent.evaluation.result import record_operator_outcome - - -@pytest.mark.parametrize("outcome", ["success", "failure"]) -def test_operator_outcome_preserves_evidence_and_refuses_overwrite(tmp_path, outcome): - verdict = {"operator_verdict": outcome, "notes": "operator evidence"} - messages = [{"role": "user", "content": "keep the completed part"}] - kwargs = { - "memory_root": tmp_path, - "recipe_tag": "dual_franka_t5", - "run_name": "run-1", - "session_number": 2, - "state_output_dir": tmp_path / "session_002", - "verdict": verdict, - "messages": messages, - "planner_error": "interrupted by verdict", - } - path = record_operator_outcome(**kwargs) - assert path.name == f"run-1-session-002-{outcome}.json" - record = json.loads(path.read_text()) - assert record["verdict"] == verdict - assert record["messages"] == messages - assert record["session"] == 2 - assert record["evidence_dir"] == str(tmp_path / "session_002") - assert record["planner_error"] == "interrupted by verdict" - with pytest.raises(FileExistsError): - record_operator_outcome(**kwargs) - assert json.loads(path.read_text()) == record def test_write_json_atomic_replaces_complete_record(tmp_path: Path) -> None: diff --git a/tests/unit_tests/rpent/planner/test_claude_contracts.py b/tests/unit_tests/rpent/planner/test_claude_contracts.py index 30dc59d56..845d489e8 100644 --- a/tests/unit_tests/rpent/planner/test_claude_contracts.py +++ b/tests/unit_tests/rpent/planner/test_claude_contracts.py @@ -17,24 +17,20 @@ import asyncio import json import queue -import threading from pathlib import Path from typing import Any import claude_agent_sdk import pytest -from rpent.cli.tui import next_user_line from rpent.dashboard.events import TranscriptEvent, UsageEvent from rpent.planner.claude_code import ( ClaudeCodePlanner, _build_rpent_server, _ClaudeSessionDriver, _Recorder, - _TerminalSessionAdapter, _tool_result_to_mcp, ) -from rpent.session.input import SessionInputQueue from rpent.tools import ToolResult, tool from rpent.tools.common import CommonTools @@ -592,49 +588,3 @@ async def close(self): @pytest.fixture(autouse=True) def tool_template_output_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("rpent.utils.templates.get_output_dir", lambda: tmp_path) - - -def test_terminal_adapter_cleanup_retires_delayed_reader( - monkeypatch: pytest.MonkeyPatch, -) -> None: - source = queue.Queue() - attempt = SessionInputQueue(source) - started = threading.Event() - release = threading.Event() - emitted = [] - queries = [] - - def delayed_read(input_queue): - started.set() - assert release.wait(1) - return next_user_line(input_queue) - - monkeypatch.setattr("rpent.planner.claude_code.next_user_line", delayed_read) - - class Driver: - async def interrupt(self): - pass - - async def query(self, text): - queries.append(text) - - async def run(): - adapter = _TerminalSessionAdapter( - toolkit=FakeToolkit(), input_queue=attempt, emit_user=emitted.append - ) - pump = asyncio.create_task(adapter.run(Driver())) - try: - assert await asyncio.to_thread(started.wait, 1) - source.put("feedback for the next reply") - await adapter.close() - finally: - release.set() - await asyncio.wait_for(pump, timeout=1) - - asyncio.run(run()) - - assert emitted == [] - assert queries == [] - assert not attempt.cancelled - assert attempt.get(block=False) == "feedback for the next reply" - assert source.empty() diff --git a/tests/unit_tests/rpent/planner/test_codex_contracts.py b/tests/unit_tests/rpent/planner/test_codex_contracts.py index 4b8076bc9..1ccaf471c 100644 --- a/tests/unit_tests/rpent/planner/test_codex_contracts.py +++ b/tests/unit_tests/rpent/planner/test_codex_contracts.py @@ -338,11 +338,9 @@ def codex_config(**kwargs: Any) -> dict[str, Any]: assert os.environ["no_proxy"] == ".corp.invalid" -@pytest.mark.parametrize("effort", ["none", "ultra"]) def test_planner_forwards_configured_service_tier( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - effort: str, ) -> None: install_fake_backend(monkeypatch) monkeypatch.setenv("CODEX_SERVICE_TIER", "fast") @@ -353,14 +351,7 @@ def test_planner_forwards_configured_service_tier( }, ] - CodexPlanner( - output_dir=str(tmp_path), - repo_root=tmp_path, - model="gpt-6-astra", - reasoning_effort=effort, - timeout_s=1, - dashboard_events=RecordingSink(), - ).solve( + make_planner(tmp_path, RecordingSink()).solve( system_prompt="system rules", user_message="user task", toolkit=FakeToolkit(), @@ -368,9 +359,7 @@ def test_planner_forwards_configured_service_tier( ) fake_codex = FakeCodex.instances[0] - assert fake_codex.thread_options["model"] == "gpt-6-astra" assert fake_codex.thread_options["service_tier"] == "fast" - assert fake_codex.thread.turn_prompts[0][1]["effort"].value == effort assert fake_codex.thread.turn_prompts[0][1]["service_tier"] == "fast" @@ -1240,53 +1229,7 @@ def test_retry_error_is_visible_without_poisoning_successful_turn(): assert "Model connection failed" in str(sink.events[-1]) -def test_reply_cleanup_returns_feedback_dequeued_before_cancellation( - tmp_path, monkeypatch -): - from rpent.cli.tui import next_user_line - from rpent.session.input import SessionInputQueue - - install_fake_backend(monkeypatch) - dequeued = threading.Event() - cancelled = threading.Event() - cancel = SessionInputQueue.cancel - - def read(scope): - line = next_user_line(scope) - dequeued.set() - assert cancelled.wait(2) - return line - - def cancel_scope(scope): - cancel(scope) - cancelled.set() - - def stream(turn): - assert dequeued.wait(2) - yield from turn.events - - monkeypatch.setattr(codex_module, "next_user_line", read) - monkeypatch.setattr(SessionInputQueue, "cancel", cancel_scope) - monkeypatch.setattr(FakeTurn, "stream", stream) - inputs = queue.Queue() - inputs.put("Feedback entered as the reply ended") - result = make_planner(tmp_path, RecordingSink()).solve( - system_prompt="test", - user_message="task", - toolkit=FakeToolkit(), - max_turns=10, - input_queue=inputs, - ) - assert result.error is None - turn = FakeCodex.instances[-1].thread.fake_turn - assert turn.steered == [] - assert turn.interrupt_calls == 0 - assert inputs.get_nowait() == "Feedback entered as the reply ended" - assert inputs.empty() - - -@pytest.mark.parametrize("end_input", [None, "/quit", "/exit"]) -def test_terminal_eof_and_quit_interrupt_codex(tmp_path, monkeypatch, end_input): +def test_terminal_eof_interrupts_codex(tmp_path, monkeypatch): install_fake_backend(monkeypatch) interrupted = threading.Event() @@ -1301,7 +1244,7 @@ def stream(turn): monkeypatch.setattr(FakeTurn, "interrupt", interrupt) monkeypatch.setattr(FakeTurn, "stream", stream) inputs = queue.Queue() - inputs.put(end_input) + inputs.put(None) result = make_planner(tmp_path, RecordingSink()).solve( system_prompt="test", user_message="task", @@ -1312,16 +1255,3 @@ def stream(turn): assert result.error is None assert FakeCodex.instances[-1].thread.fake_turn.interrupt_calls == 1 assert inputs.empty() - - -def test_zero_timeout_disables_deadline(tmp_path, monkeypatch): - install_fake_backend(monkeypatch) - planner = make_planner(tmp_path, RecordingSink(), timeout_s=0) - assert planner._timeout_s is None - result = planner.solve( - system_prompt="test", user_message="task", toolkit=FakeToolkit(), max_turns=10 - ) - assert result.error is None - assert FakeCodex.instances[-1].thread.fake_turn.interrupt_calls == 0 - with pytest.raises(ValueError, match="nonnegative"): - make_planner(tmp_path, RecordingSink(), timeout_s=-1) From 85b73b8fe3ad5552dc2b77d013afeefb99d8e5c7 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 23:13:26 +0800 Subject: [PATCH 07/18] test(dashboard): keep one complete operator-controlled task flow --- .../dual_franka/test_dashboard_operator.py | 209 ++++-------------- .../rpent/dashboard/test_server_contracts.py | 17 -- .../rpent/dashboard/test_session_contracts.py | 76 ------- 3 files changed, 38 insertions(+), 264 deletions(-) diff --git a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py index 766fb7d23..92074d49e 100644 --- a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py +++ b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py @@ -12,173 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Web operator contracts using real Dashboard, toolkit and memory components.""" +"""Offline check of an operator-controlled task through the web API.""" -import json import time from concurrent.futures import ThreadPoolExecutor from dataclasses import replace from types import SimpleNamespace -import pytest from fastapi.testclient import TestClient from robots.dual_franka import robot_spec -from robots.dual_franka.toolkit import DualFrankaToolkit -from rpent.dashboard.events import NullDashboardEventSink -from rpent.dashboard.operator import DashboardOperator +from rpent.cli import dashboard, main from rpent.dashboard.server import DashboardServer from rpent.dashboard.state import DashboardState -from rpent.memory import MemoryManager from rpent.planner.base import PlannerResult from tests.unit_tests.robots.dual_franka.test_exploration import FakeEnv -def call(toolkit, name, **kwargs): - return toolkit.execute_tool(name, kwargs).data - - -def wait_for_request(operator): - deadline = time.monotonic() + 3 - while operator.snapshot() is None and time.monotonic() < deadline: - time.sleep(0.01) - assert operator.snapshot() is not None - - -@pytest.fixture -def pending_reset(tmp_path, monkeypatch, dual_franka_robot_config): - from robots.franka.runtime_config import set_robot_config_path - - set_robot_config_path(dual_franka_robot_config) - monkeypatch.setattr("robots.dual_franka.toolkit.get_output_dir", lambda: tmp_path) - env = FakeEnv() - state = DashboardState( - output_dir=tmp_path, dashboard_spec=robot_spec.DUAL_FRANKA_DASHBOARD_SPEC - ) - state.shared_services_ready() - state.request_task({"task_id": 3}) - state.wait_for_task(0) - operator = DashboardOperator(tmp_path) - state.operator = operator - toolkit = DualFrankaToolkit( - runtime_kwargs={"env": env, "model": None, "task_description": "test"}, - dashboard_events=NullDashboardEventSink(), - memory=MemoryManager( - tmp_path / "memory", memory_access="inbox_write", inbox_cell_tag="test" - ), - mode="evaluation", - state_output_dir=tmp_path, - operator_input=state.operator, - ) - state.bind_toolkit(toolkit) - with ( - TestClient(DashboardServer(state=state)._app) as client, - ThreadPoolExecutor() as pool, - ): - future = pool.submit(call, toolkit, "request_scene_reset", reason="web reset") - try: - wait_for_request(state.operator) - pending = client.get("/api/session/state").json()["operator"] - yield toolkit, env, state, client, pending, future - finally: - operator.close() - toolkit.close() - state.unbind_toolkit(toolkit) - - -def message(pending, text): - return { - "text": text, - "operator_context": { - "generation": pending["generation"], - "request_id": pending["pending"]["id"], - }, - } - - -@pytest.mark.parametrize("answer", ["done", "abort"]) -def test_reset_waits_for_one_matching_reply(pending_reset, answer): - toolkit, env, _, client, pending, future = pending_reset - reply = message(pending, f"/{answer} scene checked") - for context in ( - {"generation": -1}, - {**reply["operator_context"], "request_id": "stale"}, - ): - assert ( - client.post( - "/api/session/messages", - json={"text": "/done", "operator_context": context}, - ).status_code - == 422 - ) - assert not future.done() and env.resets == 0 - assert client.post("/api/session/messages", json=reply).status_code == 202 - result = future.result(3) - assert env.resets == int(answer == "done") - if answer == "done": - assert toolkit.state.latest_record().result["operator_notes"] == "scene checked" - else: - assert result["code"] == "tool_cancelled" - assert client.post("/api/session/messages", json=reply).status_code == 422 - assert env.resets == int(answer == "done") - - -def test_verdict_closes_only_current_task(pending_reset): - toolkit, env, state, client, pending, future = pending_reset - assert ( - client.post("/api/session/messages", json=message(pending, "/done")).status_code - == 202 - ) - future.result(3) - reply = message(pending, "/success checked") - assert client.post("/api/session/messages", json=reply).status_code == 202 - assert call(toolkit, "open_gripper", arm="right")["motion_refused"] - toolkit.finalize_direct_verdict() - assert toolkit.solved() - assert client.post("/api/session/messages", json=reply).status_code == 422 - state.unbind_toolkit(toolkit) - state.complete_task(state="cancelled") - assert client.get("/api/session/state").json()["state"] == "succeeded" - state.request_task({"task_id": 4}) - state.wait_for_task(0) - state.bind_toolkit(toolkit) - assert client.post("/api/session/messages", json=reply).status_code == 409 - assert env.resets == 1 - - -def test_late_verdict_preserves_queued_task(pending_reset): - toolkit, _, state, client, pending, _ = pending_reset - assert ( - client.post( - "/api/session/messages", json=message(pending, "/rpent-task 4") - ).status_code - == 202 - ) - assert ( - client.post( - "/api/session/messages", json=message(pending, "/success") - ).status_code - == 422 - ) - assert not toolkit.direct_verdict_requested - state.unbind_toolkit(toolkit) - state.complete_task(state="cancelled") - assert state.wait_for_task(0).request == {"task_id": 4} - - -def test_cancel_releases_operator_wait_without_reset(pending_reset): - toolkit, env, state, _, _, future = pending_reset - toolkit.cancel_active_and_wait() - future.result(3) - assert env.resets == 0 and state.operator.snapshot() is None - - -@pytest.mark.parametrize("verdict", ["success", "failure", "abort"]) -def test_task_uses_web_operator_and_publishes_only_confirmed_success( - tmp_path, monkeypatch, dual_franka_robot_config, verdict +def test_web_operator_confirms_reset_and_completes_task( + tmp_path, monkeypatch, dual_franka_robot_config ): - from rpent.cli import dashboard, main - parser = main._build_argparser() robot_spec._add_cli_args(parser, use_dashboard=True) args = parser.parse_args( @@ -216,16 +69,40 @@ def test_task_uses_web_operator_and_publishes_only_confirmed_success( def solve(**kwargs): toolkit = kwargs["toolkit"] assert env.resets == 0 - with ThreadPoolExecutor() as pool: - future = pool.submit(call, toolkit, "request_scene_reset", reason="test") - wait_for_request(state.operator) - pending = state.operator_snapshot() - state.operator_reply( - pending["generation"], pending["pending"]["id"], "done" + with ( + TestClient(DashboardServer(state=state)._app) as client, + ThreadPoolExecutor() as pool, + ): + future = pool.submit( + toolkit.execute_tool, "request_scene_reset", {"reason": "test"} + ) + deadline = time.monotonic() + 3 + while state.operator.snapshot() is None and time.monotonic() < deadline: + time.sleep(0.01) + operator = client.get("/api/session/state").json()["operator"] + assert operator["pending"] is not None and env.resets == 0 + context = { + "generation": operator["generation"], + "request_id": operator["pending"]["id"], + } + assert ( + client.post( + "/api/session/messages", + json={"text": "/done", "operator_context": context}, + ).status_code + == 202 + ) + assert not future.result(3).is_error + toolkit.execute_tool( + "move_delta", {"arm": "right", "delta_xyz": [0, 0, 0.01]} + ) + assert ( + client.post( + "/api/session/messages", + json={"text": "/success", "operator_context": context}, + ).status_code + == 202 ) - future.result(3) - call(toolkit, "move_delta", arm="right", delta_xyz=[0, 0, 0.01]) - state.operator_finish(pending["generation"], verdict, "checked") return PlannerResult() monkeypatch.setattr( @@ -244,14 +121,4 @@ def solve(**kwargs): is None ) assert env.resets == 1 and len(env.moves) == 1 - for filename in ("dual_franka_t3.json", "dual_franka_t3_recipe.jsonl"): - assert (tmp_path / "memory/task-specific" / filename).is_file() is ( - verdict == "success" - ) - if verdict != "abort": - evidence = list((tmp_path / "memory").rglob(f"*-{verdict}.json")) - assert len(evidence) == 1 - assert ( - json.loads(evidence[0].read_text())["verdict"]["operator_verdict"] - == verdict - ) + assert (tmp_path / "memory/task-specific/dual_franka_t3_recipe.jsonl").is_file() diff --git a/tests/unit_tests/rpent/dashboard/test_server_contracts.py b/tests/unit_tests/rpent/dashboard/test_server_contracts.py index 5a54023f1..340def908 100644 --- a/tests/unit_tests/rpent/dashboard/test_server_contracts.py +++ b/tests/unit_tests/rpent/dashboard/test_server_contracts.py @@ -242,20 +242,3 @@ def test_healthz_still_reports_transport_liveness_only(state: DashboardState) -> assert resp.status_code == 200 assert resp.json() == {"ok": True} - - -def test_generic_messages_do_not_enable_operator_control(state): - state.shared_services_ready() - state.request_task({"seed": 1}) - state.wait_for_task(timeout=0) - state.set_planner_activity("idle", accepting_input=True) - client = _client(_server(state)) - for text in ("/done", "/success"): - response = client.post( - "/api/session/messages", - json={"text": text, "operator_context": {"generation": -1}}, - ) - assert response.status_code == 202 - assert state.claim_next_pending_message().text == text - assert client.get("/api/session/state").json()["operator"]["enabled"] is False - assert not state.task_replacement_requested diff --git a/tests/unit_tests/rpent/dashboard/test_session_contracts.py b/tests/unit_tests/rpent/dashboard/test_session_contracts.py index 2a53e6914..5aa1ef2a3 100644 --- a/tests/unit_tests/rpent/dashboard/test_session_contracts.py +++ b/tests/unit_tests/rpent/dashboard/test_session_contracts.py @@ -318,79 +318,3 @@ def solve(self, **kwargs: Any) -> PlannerResult: ) else: assert state.warnings == [] - - -@pytest.mark.parametrize("environment_success", [False, True]) -def test_external_robot_dashboard_keeps_its_own_operator_and_result_contract( - tmp_path, monkeypatch, environment_success -): - from robots.yam.robot_spec import YAM_DASHBOARD_SPEC, get_robot_spec - from rpent.cli import dashboard as dashboard_cli - from rpent.dashboard.state import DashboardState - - config = RunConfig( - recipe_tag="yam_t0", - output_dir=tmp_path / "task", - prompt_vars={}, - task_desc={"robot": "yam"}, - ) - finalized = [] - closed = [] - toolkit = SimpleNamespace( - memory=None, - solved=lambda: environment_success, - write_recipe=lambda recipe_tag: "", - cancel_active_and_wait=lambda: None, - close=lambda: closed.append(True), - ) - - def get_toolkit(*args, **kwargs): - assert "operator_input" not in kwargs - assert kwargs["mode"] == "evaluation" - return toolkit - - def solve(**kwargs): - assert "DASHBOARD OPERATOR" not in kwargs["user_message"] - return PlannerResult(finish_result={"success": True}, messages=[], stats={}) - - from dataclasses import replace - - spec = replace( - get_robot_spec(), - parse_config=lambda args: config, - init_runtime=lambda *args: ([], {}), - prepare_memory=lambda args, run_config: None, - prompts=PromptBundle( - system=lambda variables: "system", user=lambda variables: "user" - ), - finalize_run=finalized.append, - ) - from rpent.cli.main import _build_argparser - - args = _build_argparser().parse_args(["--robot", "yam", "--planner", "api"]) - args.robot_name = "yam" - args.auto_merge_memory = False - state = DashboardState(output_dir=tmp_path, dashboard_spec=YAM_DASHBOARD_SPEC) - monkeypatch.setattr(dashboard_cli, "get_toolkit", get_toolkit) - monkeypatch.setattr( - dashboard_cli, - "build_planner", - lambda *args, **kwargs: SimpleNamespace(solve=solve), - ) - - error = dashboard_cli._run_dashboard_task( - args=args, - robot_spec=spec, - state=state, - claimed=ClaimedTask(number=1, request={}, output_dir=config.output_dir), - shared_runtime_kwargs={}, - unique_components=set(), - session_root=tmp_path / "session", - ) - - assert error is None - assert state.operator is None - assert closed == [True] - assert len(finalized) == 1 - assert finalized[0].environment_success is environment_success - assert finalized[0].agent_error is None From 0515c3b61adcb1ab0e99aeba912463b2ccb5589f Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 23:22:19 +0800 Subject: [PATCH 08/18] test(observation): exercise public acquisition and segmentation paths --- .../dual_franka/test_dual_franka_tools.py | 18 +++++------------- .../robots/dual_franka/test_exploration.py | 15 +-------------- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py b/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py index 596bd538e..cc97ebb1a 100644 --- a/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py +++ b/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py @@ -700,16 +700,12 @@ def test_segment_returns_mask_overlay_and_world_point(tmp_path: Path): sam3_client=FakeSam3Client(), ) result = result_native.data - from robots.dual_franka.perception import _mask_to_camera_world - - rejected = _mask_to_camera_world( - np.ones((8, 8), dtype=bool), - np.full((8, 8), 3.958), - camera="d455", + state.save("d455_depth.npy", np.full((8, 8), 3.958), step=step) + rejected = segment( + prompt="white cardboard box interior", state=state, - step_idx=step, - min_valid=1, - ) + sam3_client=FakeSam3Client(), + ).data finally: set_robot_config_path(None) @@ -726,10 +722,6 @@ def test_segment_returns_mask_overlay_and_world_point(tmp_path: Path): assert result_native.images and all(result_native.images) assert rejected["point_xyz"] is None - assert rejected["valid_localization_pixels"] == 0 - assert rejected["raw_depth_median_m"] == 3.958 - assert rejected["mask_bbox_rc"] == [0, 0, 7, 7] - assert len(rejected["centroid_pixel"]) == 2 assert result["image_block_order"] == ["d455_segment_overlay"] blocks = mcp_result(result_native)["content"] diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index 970a01595..6add97464 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -25,7 +25,6 @@ from robots.dual_franka.toolkit import DualFrankaToolkit from rpent.dashboard.events import NullDashboardEventSink from rpent.memory import MemoryManager -from rpent.tools import ToolResult class FakeEnv: @@ -689,13 +688,11 @@ def test_explore_task_is_only_in_user_prompt(): assert section in user -def test_current_view_refreshes_but_historical_view_does_not(setup, monkeypatch): +def test_current_view_refreshes_but_historical_view_does_not(setup): t, env, replies = setup reset(t, replies) old_step = t.state.latest_step reads = [] - published = [] - monkeypatch.setattr(t, "_publish_step", published.append) original = env.get_observation def observe(): @@ -708,21 +705,11 @@ def observe(): observed = t.execute_tool("view_env_state", {}) assert len(reads) == 1 and t.state.latest_step == old_step + 1 assert observed.images == [t.state.load_bytes("d455.png")] - assert len(published) == 1 - assert published[0].command == {"action": "observe_current"} assert np.all(t.state.load("d455.png") == 77) call(t, "view_env_state", step=old_step) assert len(reads) == 1 - assert ( - "error" in t._current_perception(lambda **kw: ToolResult(), step=old_step).data - ) - t._current_perception(lambda **kw: ToolResult(), step=t.state.latest_step) - assert len(reads) == 1 # Localization must not silently acquire another frame. - assert len(published) == 1 assert env.resets == 1 and not env.moves call(t, "move_delta", arm="left", delta_xyz=[0.01, 0, 0]) assert len(reads) == 2 and t.state.latest_step == old_step + 2 - assert len(published) == 2 - assert published[-1].command["action"] == "move_delta" assert len(env.moves) == 1 and env.moves[0][0] == "left" np.testing.assert_allclose(env.moves[0][1], [0.01, 0, 0]) From af5b7abfb5878425072d5568877334b864a43bdc Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 23:22:53 +0800 Subject: [PATCH 09/18] test(interaction): fold feedback coverage into the existing input flow --- .../rpent/tools/test_human_in_the_loop.py | 40 +++++-------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/tests/unit_tests/rpent/tools/test_human_in_the_loop.py b/tests/unit_tests/rpent/tools/test_human_in_the_loop.py index cfe5fbcbe..c0878086c 100644 --- a/tests/unit_tests/rpent/tools/test_human_in_the_loop.py +++ b/tests/unit_tests/rpent/tools/test_human_in_the_loop.py @@ -18,14 +18,16 @@ import pytest -from rpent.tools.human_in_the_loop import ( - HumanInTheLoopInput, -) +from rpent.tools.human_in_the_loop import HumanInTheLoopInput from rpent.tools.toolkit import ToolCancelled -def test_operator_replies_are_request_scoped_and_do_not_consume_steering(monkeypatch): - broker = HumanInTheLoopInput(interactive=True) +def test_operator_replies_are_request_scoped_and_do_not_consume_steering( + tmp_path, monkeypatch +): + broker = HumanInTheLoopInput( + interactive=True, feedback_path=tmp_path / "feedback.jsonl" + ) ready = threading.Event() result = [] monkeypatch.setattr("builtins.print", lambda *args, **kwargs: ready.set()) @@ -43,31 +45,9 @@ def test_operator_replies_are_request_scoped_and_do_not_consume_steering(monkeyp assert result == ["done"] and not worker.is_alive() assert broker.route_line(f"/operator {request_id} success") assert broker._pending is None - - -@pytest.mark.parametrize("kind,command", [("reset", "/done"), ("verdict", "/continue")]) -def test_feedback_notes_are_returned_and_preserved( - tmp_path, monkeypatch, kind, command -): - broker = HumanInTheLoopInput( - interactive=True, feedback_path=tmp_path / "feedback.jsonl" - ) - ready = threading.Event() - result = [] - monkeypatch.setattr("builtins.print", lambda *args, **kwargs: ready.set()) - worker = threading.Thread( - target=lambda: result.append(broker.request("confirm", lambda: None, kind=kind)) - ) - worker.start() - assert ready.wait(2) - assert not broker.route_line("保留前半段成果，重点练后半段") - assert not result # Feedback alone never authorizes robot reset. - assert broker.route_line(command + " 篮筐中的碗盘保持不动") - worker.join(2) - assert result == [command[1:] + " 篮筐中的碗盘保持不动"] - assert "重点练后半段" in broker.feedback_prompt() - assert "篮筐中的碗盘保持不动" in broker.feedback_prompt() - assert len((tmp_path / "feedback.jsonl").read_text().splitlines()) == 2 + assert not broker.route_line("keep the current progress") + assert "keep the current progress" in broker.feedback_prompt() + assert "keep the current progress" in (tmp_path / "feedback.jsonl").read_text() def test_operator_eof_and_cancellation_release_pending_requests(monkeypatch): From c4317792f5ec1148ebcf38b009348715ed73904e Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 10 Oct 2026 23:49:50 +0800 Subject: [PATCH 10/18] refactor(interaction): reuse lifecycle guards and narrow operator routing --- robots/dual_franka/toolkit.py | 21 +------ rpent/cli/attended.py | 28 +++------ rpent/cli/dashboard.py | 2 +- rpent/dashboard/operator.py | 25 +------- rpent/dashboard/state.py | 62 ++++++------------- rpent/session/input.py | 30 ++------- .../dual_franka/test_dashboard_operator.py | 9 +++ .../robots/dual_franka/test_exploration.py | 3 +- 8 files changed, 50 insertions(+), 130 deletions(-) diff --git a/robots/dual_franka/toolkit.py b/robots/dual_franka/toolkit.py index eb26226ed..639606228 100644 --- a/robots/dual_franka/toolkit.py +++ b/robots/dual_franka/toolkit.py @@ -99,7 +99,6 @@ def __init__( self._direct_verdict_event = threading.Event() self._direct_verdict: str | None = None self._direct_verdict_notes = "" - self._session_closed = threading.Event() super().__init__( runtime_kwargs=runtime_kwargs, dashboard_events=dashboard_events, @@ -117,9 +116,7 @@ def request_direct_verdict(self, verdict: str, notes: str = "") -> bool: if verdict not in {"success", "failure", "abort"}: raise ValueError("verdict must be success, failure or abort") with self._scheduler.condition: - if self._session_closed.is_set(): - return False - if self._direct_verdict_event.is_set(): + if self._scheduler.closed: return False if not self._attended or ( verdict == "success" @@ -133,20 +130,14 @@ def request_direct_verdict(self, verdict: str, notes: str = "") -> bool: return True def raise_if_cancelled(self) -> None: - if self._session_closed.is_set(): - raise ToolCancelled("exploration session closed") if self._direct_verdict_event.is_set(): raise ToolCancelled( "operator submitted a terminal verdict; stopping exploration" ) super().raise_if_cancelled() - def execute_tool( - self, - name: str, - input_dict: dict[str, Any], - ) -> ToolResult: - if self._direct_verdict_event.is_set() or self._session_closed.is_set(): + def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: + if self._direct_verdict_event.is_set(): return ToolResult( data={ "error": ( @@ -157,12 +148,6 @@ def execute_tool( ) return super().execute_tool(name, input_dict) - def close(self) -> None: - """Stop and join old tools before their operator input can be reused.""" - self._session_closed.set() - self.cancel_active_and_wait() - super().close() - def finalize_direct_verdict(self) -> dict[str, Any]: """Record fresh operator evidence after all active work has stopped.""" self.cancel_active_and_wait() diff --git a/rpent/cli/attended.py b/rpent/cli/attended.py index a03519235..b9d79686d 100644 --- a/rpent/cli/attended.py +++ b/rpent/cli/attended.py @@ -16,7 +16,7 @@ import json import shutil -import time +from contextlib import ExitStack from pathlib import Path from rpent.cli.tui import next_user_line @@ -79,14 +79,15 @@ def _solve_attended( planner, *, operator_input, state_output_dir, keep_mcp_alive=False, **kwargs ): """Own the transport for the whole attended attempt, including human waits.""" - server = None - if keep_mcp_alive: - from rpent.planner.utils.http_mcp_server import HttpMcpServer + with ExitStack() as cleanup: + if keep_mcp_alive: + from rpent.planner.utils.http_mcp_server import HttpMcpServer - server = HttpMcpServer(kwargs["toolkit"]) - kwargs["mcp_server"] = server - try: - if server is not None: + server = HttpMcpServer(kwargs["toolkit"]) + kwargs["mcp_server"] = server + cleanup.callback(server.stop) + # Release pending tool handlers before shutting down HTTP. + cleanup.callback(kwargs["toolkit"].cancel_active_and_wait) server.start() return _solve_attended_turns( planner, @@ -94,11 +95,6 @@ def _solve_attended( state_output_dir=state_output_dir, **kwargs, ) - finally: - if server is not None: - # Release pending tool handlers before asking HTTP to shut down. - kwargs["toolkit"].cancel_active_and_wait() - server.stop() def _solve_attended_turns(planner, *, operator_input, state_output_dir, **kwargs): @@ -147,12 +143,6 @@ def _solve_attended_turns(planner, *, operator_input, state_output_dir, **kwargs logger.info( "Waiting for the current operator request; keeping this attempt alive." ) - while ( - operator_input.pending_kind is not None - and not toolkit.direct_verdict_requested - and not operator_input.closed - ): - time.sleep(0.1) toolkit.wait_active() if toolkit.direct_verdict_requested: break diff --git a/rpent/cli/dashboard.py b/rpent/cli/dashboard.py index b1ed947d2..f1b7e80c8 100644 --- a/rpent/cli/dashboard.py +++ b/rpent/cli/dashboard.py @@ -282,7 +282,7 @@ def _run_dashboard_task( from rpent.dashboard.operator import DashboardOperator state.operator = DashboardOperator( - state_output_dir, on_change=state.operator_changed + on_change=state.operator_changed ) operator_kwargs["operator_input"] = state.operator toolkit = get_toolkit( diff --git a/rpent/dashboard/operator.py b/rpent/dashboard/operator.py index a066b9b5c..8b1b4cb26 100644 --- a/rpent/dashboard/operator.py +++ b/rpent/dashboard/operator.py @@ -16,40 +16,25 @@ from __future__ import annotations -import json import threading import uuid from collections.abc import Callable -from pathlib import Path from typing import Any from rpent.tools.toolkit import ToolCancelled class DashboardOperator: - """Own one pending confirmation and archive its request and reply.""" + """Own one pending confirmation for the active Dashboard task.""" - def __init__( - self, output_dir: str | Path, on_change: Callable[[], None] = lambda: None - ) -> None: - """Initialize the confirmation channel. - - Args: - output_dir: Directory for the operator event log. - on_change: Callback invoked when a request opens or closes. - """ + def __init__(self, on_change: Callable[[], None] = lambda: None) -> None: + """Notify on_change when a confirmation opens or closes.""" self._condition = threading.Condition() self._pending: dict[str, Any] | None = None self._answer: str | None = None self._closed = False - self._path = Path(output_dir) / "dashboard_operator.jsonl" self._on_change = on_change - def _record(self, event: dict[str, Any]) -> None: - self._path.parent.mkdir(parents=True, exist_ok=True) - with self._path.open("a") as file: - file.write(json.dumps(event, ensure_ascii=False) + "\n") - def snapshot(self) -> dict[str, Any] | None: """Return the pending request, or None when no confirmation is pending.""" with self._condition: @@ -93,7 +78,6 @@ def request( "choices": choices, } self._answer = None - self._record({"event": "request", **self._pending}) self._on_change() try: with self._condition: @@ -132,9 +116,6 @@ def reply(self, request_id: str, answer: str, notes: str = "") -> None: if answer not in self._pending["choices"] or not isinstance(notes, str): raise ValueError("invalid operator response") self._answer = answer + (" " + notes.strip() if notes.strip() else "") - self._record( - {"event": "reply", "id": request_id, "answer": answer, "notes": notes} - ) self._condition.notify_all() def close(self) -> None: diff --git a/rpent/dashboard/state.py b/rpent/dashboard/state.py index 3d4148a7c..77875999b 100644 --- a/rpent/dashboard/state.py +++ b/rpent/dashboard/state.py @@ -253,37 +253,17 @@ def operator_snapshot(self): "verdict": self.operator_verdict, } - def operator_reply(self, generation, request_id, answer, notes=""): - with self._condition: - if ( - generation != self._task_generation - or self.operator is None - or self._session_state == "switch_pending" - ): - raise ValueError("expired operator task") - self.operator.reply(request_id, answer, notes) - - def operator_finish(self, generation, verdict, notes=""): - with self._condition: - if verdict not in {"success", "failure", "abort"} or not isinstance( - notes, str - ): - raise ValueError("invalid operator verdict") - if ( - generation != self._task_generation - or self.operator is None - or self._toolkit is None - or self._session_state == "switch_pending" - ): - raise ValueError("no matching active operator task") - if not self._toolkit.request_direct_verdict(verdict, notes): - raise ValueError("operator verdict refused in current state") - self.operator_verdict = verdict - self._pending_task = None - self._session_state = "switch_pending" - self._accepting_input = False - self.operator.close() - self._interaction_changed_locked() + def _finish_operator_locked(self, verdict: str, notes: str) -> None: + if self._toolkit is None: + raise ValueError("no matching active operator task") + if not self._toolkit.request_direct_verdict(verdict, notes): + raise ValueError("operator verdict refused in current state") + self.operator_verdict = verdict + self._pending_task = None + self._session_state = "switch_pending" + self._accepting_input = False + self.operator.close() + self._interaction_changed_locked() def bind_toolkit(self, toolkit: Toolkit) -> None: """Expose a TaskRun Toolkit through the Dashboard primitive API.""" @@ -554,30 +534,24 @@ def submit_input( if parsed is not None: command = f"/{parsed.name}" notes = parsed.notes - current = self.operator_snapshot() - if operator_context is None: - raise ValueError( - "No active operator channel; refresh the Dashboard." - ) if not isinstance(operator_context, dict): raise ValueError("operator_context must be an object.") - if operator_context.get("generation") != current["generation"]: + if ( + operator_context.get("generation") != self._task_generation + or self._session_state == "switch_pending" + ): raise ValueError("expired operator task") - pending = current["pending"] + pending = self.operator.snapshot() if command == "/continue" and pending is None: return self._submit_message(text) if command in {"/success", "/failure", "/abort"}: - self.operator_finish(current["generation"], command[1:], notes) + self._finish_operator_locked(parsed.name, notes) else: request_id = operator_context.get("request_id") answer = command[1:] if command == "/operator": request_id, answer = parsed.request_id, parsed.answer - if pending is None or request_id != pending["id"]: - raise ValueError("No matching pending operator request.") - self.operator_reply( - current["generation"], request_id, answer, notes - ) + self.operator.reply(request_id, answer, notes) self._control_feedback = [f"{command} accepted"] self._interaction_changed_locked() return None diff --git a/rpent/session/input.py b/rpent/session/input.py index fb8b14eeb..35cf0526d 100644 --- a/rpent/session/input.py +++ b/rpent/session/input.py @@ -18,20 +18,6 @@ import queue import threading -import time -from typing import Protocol - - -class InputQueue(Protocol): - """Input operations shared by readers and planner consumers.""" - - def get(self, block: bool = True, timeout: float | None = None) -> str | None: - """Read a submitted line or the source's EOF signal.""" - ... - - def put(self, item: str | None) -> None: - """Forward a line or the source's EOF signal.""" - ... class SessionInputQueue: @@ -57,11 +43,8 @@ def cancel(self) -> None: with self._lock: self._cancelled.set() - def get(self, block: bool = True, timeout: float | None = None) -> str | None: + def get(self, block: bool = True) -> str | None: """Read input, returning None when this scope or its source closes.""" - if block and timeout is not None and timeout < 0: - raise ValueError("timeout must be a non-negative number") - deadline = time.monotonic() + timeout if timeout is not None else None while True: with self._lock: if self.cancelled: @@ -71,14 +54,11 @@ def get(self, block: bool = True, timeout: float | None = None) -> str | None: except queue.Empty: if not block: raise - wait_time = 0.1 - if deadline is not None: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise queue.Empty - wait_time = min(wait_time, remaining) - self._cancelled.wait(wait_time) + self._cancelled.wait(0.1) def put(self, item: str | None) -> None: """Forward real input to the source, including returned unread lines.""" self._source.put(item) + + +InputQueue = queue.Queue[str | None] | SessionInputQueue diff --git a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py index 92074d49e..3b157e145 100644 --- a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py +++ b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py @@ -85,6 +85,15 @@ def solve(**kwargs): "generation": operator["generation"], "request_id": operator["pending"]["id"], } + for stale in ( + {**context, "generation": context["generation"] - 1}, + {**context, "request_id": "expired"}, + ): + assert client.post( + "/api/session/messages", + json={"text": "/done", "operator_context": stale}, + ).is_error + assert env.resets == 0 assert ( client.post( "/api/session/messages", diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index 2478c49b0..e788efa77 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -602,7 +602,8 @@ def solve(self, *, toolkit, input_queue, **kwargs): call(toolkit, "request_scene_reset", reason="test") call(toolkit, "move_delta", arm="right", delta_xyz=[0.01, 0, 0]) assert handlers["line"]("/" + verdict) - assert input_queue.get(timeout=1) is None + assert input_queue.cancelled + assert input_queue.get() is None return SimpleNamespace( finish_result=None, messages=[], stats={}, error=planner_error ) From 02d34d4406987c9978dcbad53b1a8d5dd17a6596 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 01:33:34 +0800 Subject: [PATCH 11/18] refactor(interaction): reuse planner sessions and operator confirmations --- robots/dual_franka/prompt_bundle.py | 2 + robots/dual_franka/prompts/user.py | 5 + rpent/cli/attended.py | 123 +---------- rpent/cli/dashboard.py | 19 +- rpent/cli/main.py | 21 +- rpent/dashboard/operator.py | 125 ----------- rpent/planner/codex.py | 199 ++++++++---------- rpent/tools/human_in_the_loop.py | 106 ++++++---- rpent/tools/toolkit.py | 5 - .../robots/dual_franka/test_exploration.py | 138 +++++++----- .../rpent/planner/test_codex_contracts.py | 32 +-- 11 files changed, 276 insertions(+), 499 deletions(-) delete mode 100644 rpent/dashboard/operator.py diff --git a/robots/dual_franka/prompt_bundle.py b/robots/dual_franka/prompt_bundle.py index 6ace4ffbc..15968061a 100644 --- a/robots/dual_franka/prompt_bundle.py +++ b/robots/dual_franka/prompt_bundle.py @@ -53,4 +53,6 @@ def user_prompt(variables: Mapping[str, object] | None = None) -> PromptNode: "Use {{memory_inbox}} for reviewable exploration notes and " "{{output_dir}}/attempts/ for failed-attempt archives." ) + if (variables or {}).get("dashboard"): + node["OPERATOR INPUT"] = user_parts.DASHBOARD_OPERATOR return node diff --git a/robots/dual_franka/prompts/user.py b/robots/dual_franka/prompts/user.py index bb78eb4e0..6bf35634f 100644 --- a/robots/dual_franka/prompts/user.py +++ b/robots/dual_franka/prompts/user.py @@ -26,3 +26,8 @@ together with both arms' TCP/gripper/joint-health state, and execute the task conservatively with the exposed bounded tools. Auxiliary camera views are artifact views for targeted follow-up inspection only.""" + +DASHBOARD_OPERATOR = """Before the first motion call request_scene_reset and wait +for the operator's Dashboard confirmation. Final success/failure requires +request_operator_verdict or an explicit operator verdict. There is no terminal +input; use the Dashboard message input for operator replies.""" diff --git a/rpent/cli/attended.py b/rpent/cli/attended.py index b9d79686d..378f04c8e 100644 --- a/rpent/cli/attended.py +++ b/rpent/cli/attended.py @@ -14,12 +14,9 @@ """CLI orchestration for one operator-attended attempt.""" -import json import shutil -from contextlib import ExitStack from pathlib import Path -from rpent.cli.tui import next_user_line from rpent.evaluation.result import record_operator_outcome from rpent.session.transcript import _serialize_messages from rpent.utils.logging import get_logger @@ -51,12 +48,11 @@ def _finalize_operator_verdict( ) handoff = ( f"\nThe operator marked the preceding attempt as {verdict}. " - f"Before requesting any reset or motion, read {evidence}, " + f"Read {evidence}, " "inspect its evidence and write a grounded outcome summary " "to your memory inbox wip notes. Distinguish observations " "from hypotheses; never label a failed attempt as successful. " - "Then request_scene_reset and wait for a new operator " - "confirmation before attempting the task again.\n" + "Follow the robot's exploration workflow for the next attempt.\n" ) if verdict == "failure": error = None @@ -73,118 +69,3 @@ def _publish_success_recipe( memory.merge_memory( cell_tag=recipe_tag, run_state_dir=state_output_dir, solved=True ) - - -def _solve_attended( - planner, *, operator_input, state_output_dir, keep_mcp_alive=False, **kwargs -): - """Own the transport for the whole attended attempt, including human waits.""" - with ExitStack() as cleanup: - if keep_mcp_alive: - from rpent.planner.utils.http_mcp_server import HttpMcpServer - - server = HttpMcpServer(kwargs["toolkit"]) - kwargs["mcp_server"] = server - cleanup.callback(server.stop) - # Release pending tool handlers before shutting down HTTP. - cleanup.callback(kwargs["toolkit"].cancel_active_and_wait) - server.start() - return _solve_attended_turns( - planner, - operator_input=operator_input, - state_output_dir=state_output_dir, - **kwargs, - ) - - -def _solve_attended_turns(planner, *, operator_input, state_output_dir, **kwargs): - """Keep an unjudged attempt alive across planner replies and human pauses.""" - toolkit = kwargs["toolkit"] - original_message = kwargs["user_message"].removesuffix( - operator_input.feedback_prompt() - ) - messages = [] - usage = {} - turn_number = 0 - while True: - continue_revision = operator_input.continue_revision - result = planner.solve(**kwargs) - turn_number += 1 - messages.extend(result.messages) - for key in ( - "total_input_tokens", - "total_output_tokens", - "total_cached_input_tokens", - "tool_calls", - ): - usage[key] = usage.get(key, 0) + result.stats.get(key, 0) - # Preserve every reply before the planner reuses its output filenames. - archive = Path(state_output_dir) / f"dialogue_{turn_number:03d}.json" - archive.parent.mkdir(parents=True, exist_ok=True) - archive.write_text( - json.dumps( - { - "messages": _serialize_messages(result.messages), - "stats": result.stats, - "finish": result.finish_result, - "error": result.error, - }, - indent=2, - default=str, - ) - ) - if result.error or result.finish_result or toolkit.direct_verdict_requested: - break - if operator_input.closed: - toolkit.request_direct_verdict("abort", "Interactive input closed") - break - resumed_operator_request = operator_input.pending_kind is not None - if resumed_operator_request: - logger.info( - "Waiting for the current operator request; keeping this attempt alive." - ) - toolkit.wait_active() - if toolkit.direct_verdict_requested: - break - if operator_input.closed: - toolkit.request_direct_verdict("abort", "Interactive input closed") - break - resumed_operator_request |= ( - operator_input.continue_revision != continue_revision - ) - if resumed_operator_request: - reply = ( - "The pending operator request has returned or the operator already answered continue. " - "Read its recorded result and continue this attempt without waiting for the same reply again." - ) - else: - logger.info( - "Waiting for feedback or /continue; /success, /failure and /abort remain available." - ) - reply = next_user_line(kwargs["input_queue"]) - if toolkit.direct_verdict_requested: - break - if reply is None: - toolkit.request_direct_verdict("abort", "Interactive input closed") - break - if not resumed_operator_request: - observation = toolkit.execute_tool("view_env_state", {}).data - if observation.get("error"): - result.error = ( - f"Observation refresh failed after feedback: {observation['error']}" - ) - break - messages.append({"role": "user", "content": reply}) - kwargs["user_message"] = ( - original_message - + "\nContinue the SAME attempt, not a new exploration session. Robot state and " - "scene confirmation are retained; do not reset merely because the conversation resumed. " - f"Current attempt state and operator results are recorded under {state_output_dir}. " - "If the operator changed the scene, obtain fresh observations and localization. " - "Previous conversation:\n" - + json.dumps(_serialize_messages(messages), ensure_ascii=False, default=str) - + operator_input.feedback_prompt() - ) - result.messages = messages - result.stats.update(usage) - return result diff --git a/rpent/cli/dashboard.py b/rpent/cli/dashboard.py index f1b7e80c8..eda9f0982 100644 --- a/rpent/cli/dashboard.py +++ b/rpent/cli/dashboard.py @@ -228,15 +228,12 @@ def _run_dashboard_task( **task_runtime_kwargs, **shared_runtime_kwargs, } - prompt_vars = {**run_config.prompt_vars, "output_dir": output_dir} + prompt_vars = { + **run_config.prompt_vars, + "output_dir": output_dir, + "dashboard": True, + } session_message = robot_spec.prompts.render("user", variables=prompt_vars) - if human_interactive_run: - session_message += ( - "\nDASHBOARD OPERATOR: Before the first motion call request_scene_reset " - "and wait for the human's web confirmation. Final success/failure " - "requires request_operator_verdict or an explicit operator verdict. " - "There is no terminal input; do not ask for shell/TTY replies." - ) sessions = max( 1, int(getattr(task_args, "explore_sessions", 1) or 1), @@ -279,10 +276,10 @@ def _run_dashboard_task( if robot_spec.supports_exploration: operator_kwargs = {} if human_interactive_run: - from rpent.dashboard.operator import DashboardOperator + from rpent.tools.human_in_the_loop import HumanInTheLoopInput - state.operator = DashboardOperator( - on_change=state.operator_changed + state.operator = HumanInTheLoopInput( + interactive=True, on_change=state.operator_changed ) operator_kwargs["operator_input"] = state.operator toolkit = get_toolkit( diff --git a/rpent/cli/main.py b/rpent/cli/main.py index 120e77cb1..7c1372a78 100644 --- a/rpent/cli/main.py +++ b/rpent/cli/main.py @@ -47,7 +47,6 @@ from rpent.cli.attended import ( _finalize_operator_verdict, _publish_success_recipe, - _solve_attended, ) from rpent.cli.tui import ( start_first_prompt_resolver, @@ -598,18 +597,7 @@ def accept_verdict( operator_input.bind_verdict(accept_verdict) pending_operator_request = None try: - solve = planner.solve - if human_interactive_run and input_queue is not None: - from functools import partial - - solve = partial( - _solve_attended, - planner, - keep_mcp_alive=args.planner == "codex", - operator_input=operator_input, - state_output_dir=state_output_dir, - ) - result = solve( + result = planner.solve( system_prompt=system_prompt, user_message=session_msg, toolkit=toolkit, @@ -622,6 +610,12 @@ def accept_verdict( agent_error = result.error if operator_input is not None and args.interactive: pending_operator_request = operator_input.pending_kind + if operator_input.closed and not ( + finish_result or toolkit.direct_verdict_requested + ): + toolkit.request_direct_verdict( + "abort", "Interactive input closed" + ) if human_interactive_run and toolkit.direct_verdict_requested: finish_result, agent_error, verdict_handoff = ( _finalize_operator_verdict( @@ -772,6 +766,7 @@ def accept_verdict( if ( getattr(args, "explore", False) and getattr(args, "auto_merge_memory", False) + and not direct_operator_success and not agent_error and memory_manager is not None and (not human_interactive_exploration or solved) diff --git a/rpent/dashboard/operator.py b/rpent/dashboard/operator.py deleted file mode 100644 index 8b1b4cb26..000000000 --- a/rpent/dashboard/operator.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2026 The RPent Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Cancelable, request-scoped operator confirmations for physical tasks.""" - -from __future__ import annotations - -import threading -import uuid -from collections.abc import Callable -from typing import Any - -from rpent.tools.toolkit import ToolCancelled - - -class DashboardOperator: - """Own one pending confirmation for the active Dashboard task.""" - - def __init__(self, on_change: Callable[[], None] = lambda: None) -> None: - """Notify on_change when a confirmation opens or closes.""" - self._condition = threading.Condition() - self._pending: dict[str, Any] | None = None - self._answer: str | None = None - self._closed = False - self._on_change = on_change - - def snapshot(self) -> dict[str, Any] | None: - """Return the pending request, or None when no confirmation is pending.""" - with self._condition: - return dict(self._pending) if self._pending else None - - def request( - self, - prompt: str, - check_cancelled: Callable[[], None], - *, - kind: str | None = None, - ) -> str: - """Wait for an operator reply while checking cancellation. - - Args: - prompt: Instructions shown to the operator. - check_cancelled: Callback that raises when the operation is cancelled. - kind: ``reset`` permits done/abort; other kinds permit task verdicts. - - Returns: - The accepted answer followed by any operator notes. - - Raises: - ToolCancelled: The channel closes or the callback reports cancellation. - RuntimeError: Another confirmation is already pending. - """ - choices = ( - ["done", "abort"] - if kind == "reset" - else ["success", "failure", "continue", "abort"] - ) - with self._condition: - if self._closed: - raise ToolCancelled("operator channel closed") - if self._pending: - raise RuntimeError("another operator confirmation is pending") - self._pending = { - "id": uuid.uuid4().hex, - "kind": kind, - "prompt": prompt, - "choices": choices, - } - self._answer = None - self._on_change() - try: - with self._condition: - while self._answer is None: - check_cancelled() - if self._closed: - raise ToolCancelled("operator channel closed") - self._condition.wait(0.1) - check_cancelled() - return self._answer - finally: - with self._condition: - self._pending = None - self._answer = None - self._on_change() - - def reply(self, request_id: str, answer: str, notes: str = "") -> None: - """Answer the matching pending request once. - - Args: - request_id: Identifier from the pending request snapshot. - answer: One of the choices exposed by that request. - notes: Optional instructions or evidence accompanying the answer. - - Raises: - ValueError: The request is expired, answered, or the reply is invalid. - """ - with self._condition: - if ( - self._closed - or not self._pending - or request_id != self._pending["id"] - or self._answer is not None - ): - raise ValueError("expired or already answered operator request") - if answer not in self._pending["choices"] or not isinstance(notes, str): - raise ValueError("invalid operator response") - self._answer = answer + (" " + notes.strip() if notes.strip() else "") - self._condition.notify_all() - - def close(self) -> None: - """Close the channel and wake waiting callers without confirming anything.""" - with self._condition: - self._closed = True - self._condition.notify_all() diff --git a/rpent/planner/codex.py b/rpent/planner/codex.py index def6316c6..9dc0e2601 100644 --- a/rpent/planner/codex.py +++ b/rpent/planner/codex.py @@ -139,7 +139,6 @@ def solve( max_turns: int, input_queue: InputQueue | None = None, dashboard_interaction: DashboardInteractionPort | None = None, - mcp_server: HttpMcpServer | None = None, ) -> PlannerResult: """Run one or more Codex SDK turns for the given prompt.""" if input_queue is not None and dashboard_interaction is not None: @@ -147,16 +146,15 @@ def solve( "input_queue and dashboard_interaction cannot be used together" ) prompt = f"{system_prompt}\n\n{user_message}" if system_prompt else user_message - if dashboard_interaction is not None: - if mcp_server is not None: - raise ValueError("dashboard owns its MCP server") + if dashboard_interaction is not None or input_queue is not None: return asyncio.run( - self._solve_dashboard( + self._solve_interactive( prompt=prompt, initial_user_text=user_message, toolkit=toolkit, max_turns=max_turns, interaction=dashboard_interaction, + input_queue=input_queue, ) ) output_path, raw_stream_path, last_message_path = self._output_paths() @@ -168,9 +166,7 @@ def solve( # Start the in-thread MCP HTTP server so Codex can reach the # shared toolkit without spawning a subprocess. - owns_mcp_server = mcp_server is None - if owns_mcp_server: - mcp_server = HttpMcpServer(toolkit) + mcp_server = HttpMcpServer(toolkit) mcp_url = mcp_server.start() logger.info("mcp http endpoint: %s", mcp_url) @@ -194,8 +190,6 @@ def solve( recorder, state, mcp_url, - toolkit, - input_queue, ), name="codex-sdk", daemon=True, @@ -232,14 +226,13 @@ def solve( _write_jsonl(raw_f, {"type": "error", "message": error}) logger.info(rendered.rstrip()) finally: - if owns_mcp_server: - try: - toolkit.cancel_active_and_wait() - except Exception as exc: - logger.exception("Codex toolkit cleanup failed") - error = error or f"Toolkit cleanup failed: {exc}" - finally: - mcp_server.stop() + try: + toolkit.cancel_active_and_wait() + except Exception as exc: + logger.exception("Codex toolkit cleanup failed") + error = error or f"Toolkit cleanup failed: {exc}" + finally: + mcp_server.stop() elapsed = time.time() - started text = state.get("text", "") or output_path.read_text(errors="replace") @@ -291,8 +284,6 @@ def _run_session( recorder: "_Recorder", state: dict[str, Any], mcp_url: str, - toolkit: Toolkit, - input_queue: InputQueue | None = None, ) -> None: try: chunks: list[str] = [] @@ -305,85 +296,30 @@ def _run_session( open(output_path, "w") as out_f, open(raw_stream_path, "w") as raw_f, ): - write_lock = threading.Lock() - turn = thread.turn(prompt, **self._turn_options) state["turn"] = turn - steering_input: SessionInputQueue | None = None - steer_thread: threading.Thread | None = None - if input_queue is not None: - steering_input = SessionInputQueue(input_queue) - - def _steer() -> None: - while True: - nxt = next_user_line(steering_input) - if nxt is None: - if not steering_input.cancelled: - toolkit.cancel_active_and_wait() - try: - turn.interrupt() - except Exception: - pass - return - if steering_input.cancelled: - input_queue.put(nxt) - return - rendered = f"\n[user] {nxt}\n" - with write_lock: - chunks.append(rendered) - out_f.write(rendered) - out_f.flush() - logger.info(rendered.strip()) - try: - turn.steer(nxt) - logger.info( - "[feedback] delivered to active Codex turn" - ) - except Exception as e: - rendered = f"\n[codex-planner] steer failed: {e}\n" - with write_lock: - chunks.append(rendered) - out_f.write(rendered) - out_f.flush() - logger.info(rendered.strip()) - return - - steer_thread = threading.Thread( - target=_steer, - name="codex-steer", - daemon=True, - ) - steer_thread.start() - limit_reached = False - try: - for event in turn.stream(): - _write_jsonl(raw_f, _message_to_json(event)) - if rendered := recorder.observe(event): - with write_lock: - chunks.append(rendered) - out_f.write(rendered) - out_f.flush() - logger.info(rendered.strip()) - if ( - str(_get(event, "method", "")) != "turn/completed" - and not limit_reached - and recorder.finish_result is None - and recorder.turns >= recorder.max_turns - ): - limit_reached = True - try: - turn.interrupt() - except openai_codex.JsonRpcError as exc: - if exc.message != "no active turn to interrupt": - raise - # The terminal notification may still be queued. - finally: - if steering_input is not None: - steering_input.cancel() - if steer_thread is not None: - steer_thread.join(timeout=2) + for event in turn.stream(): + _write_jsonl(raw_f, _message_to_json(event)) + if rendered := recorder.observe(event): + chunks.append(rendered) + out_f.write(rendered) + out_f.flush() + logger.info(rendered.strip()) + if ( + str(_get(event, "method", "")) != "turn/completed" + and not limit_reached + and recorder.finish_result is None + and recorder.turns >= recorder.max_turns + ): + limit_reached = True + try: + turn.interrupt() + except openai_codex.JsonRpcError as exc: + if exc.message != "no active turn to interrupt": + raise + # The terminal notification may still be queued. state["text"] = "".join(chunks) if recorder.final_response is not None: @@ -391,14 +327,15 @@ def _steer() -> None: except Exception as e: state["error"] = e - async def _solve_dashboard( + async def _solve_interactive( self, *, prompt: str, initial_user_text: str, toolkit: Toolkit, max_turns: int, - interaction: DashboardInteractionPort, + interaction: DashboardInteractionPort | None, + input_queue: InputQueue | None, ) -> PlannerResult: """Run a controllable sequence of turns on one Codex thread.""" output_path, raw_stream_path, last_message_path = self._output_paths() @@ -421,6 +358,8 @@ def emit_event(event: Any) -> None: chunks.append(rendered) out_f.write(rendered) out_f.flush() + if input_queue is not None: + logger.info(rendered.strip()) def emit_user(text: str, *, initial: bool = False) -> None: display = ( @@ -430,6 +369,7 @@ def emit_user(text: str, *, initial: bool = False) -> None: chunks.append(rendered) out_f.write(rendered) out_f.flush() + logger.info(rendered.strip()) self._dashboard_events.emit( TranscriptEvent( {"type": "initial_prompt"} @@ -438,16 +378,19 @@ def emit_user(text: str, *, initial: bool = False) -> None: ) ) - control = DashboardPlannerControl( - interaction=interaction, - cancel_active_and_wait=toolkit.cancel_active_and_wait, - resume_calls=toolkit.resume_calls, - emit_user=emit_user, - emit_initial_user=lambda: emit_user( - initial_user_text, initial=True - ), - ) - session = _CodexDashboardSession( + if interaction is not None: + control = DashboardPlannerControl( + interaction=interaction, + cancel_active_and_wait=toolkit.cancel_active_and_wait, + resume_calls=toolkit.resume_calls, + emit_user=emit_user, + emit_initial_user=lambda: emit_user( + initial_user_text, initial=True + ), + ) + else: + control = _TerminalControl(input_queue, toolkit, emit_user) + session = _CodexSession( config=self._build_config(mcp_url), thread_options=self._thread_options, turn_options=self._turn_options, @@ -515,7 +458,45 @@ def _build_config(self, mcp_url: str) -> Any: ) -class _CodexDashboardSession: +class _TerminalControl: + """Forward terminal input through the same persistent Codex session driver.""" + + def __init__(self, input_queue: InputQueue, toolkit: Toolkit, emit_user) -> None: + self._input = SessionInputQueue(input_queue) + self._toolkit = toolkit + self._emit_user = emit_user + + async def start(self) -> None: + pass + + async def run(self, driver: _CodexSession) -> None: + try: + while True: + text = await asyncio.to_thread(next_user_line, self._input) + if text is None: + return + if self._input.cancelled: + self._input.put(text) + return + await driver._submit_text(text) + self._emit_user(text) + finally: + self.end() + + async def complete(self, driver: _CodexSession) -> None: + logger.info("Waiting for feedback or /continue in the same Codex conversation.") + + async def tool_completed(self, driver: _CodexSession) -> None: + pass + + def end(self) -> None: + self._input.cancel() + + async def cancel_active_toolkit(self) -> None: + await cancel_and_wait(self._toolkit.cancel_active_and_wait) + + +class _CodexSession: """Own one Codex thread and its current interruptible turn.""" def __init__( @@ -526,7 +507,7 @@ def __init__( turn_options: dict[str, Any], recorder: "_Recorder", emit_event, - control: DashboardPlannerControl, + control: DashboardPlannerControl | _TerminalControl, ) -> None: self._config = config self._thread_options = thread_options diff --git a/rpent/tools/human_in_the_loop.py b/rpent/tools/human_in_the_loop.py index 5e7a04f47..45a67f088 100644 --- a/rpent/tools/human_in_the_loop.py +++ b/rpent/tools/human_in_the_loop.py @@ -87,15 +87,22 @@ class HumanInTheLoopInput: """ - def __init__(self, *, interactive: bool, feedback_path: Path | None = None) -> None: + def __init__( + self, + *, + interactive: bool, + feedback_path: Path | None = None, + on_change: Callable[[], None] | None = None, + ) -> None: self.interactive = interactive self._feedback_path = feedback_path self._feedback: list[str] = [] self._lock = threading.Lock() self._pending: tuple[str, queue.Queue[str | None]] | None = None self._pending_kind: str | None = None + self._pending_prompt = "" + self._on_change = on_change self._closed = False - self._continue_revision = 0 self._verdict_handler: Callable[[str, str], bool] | None = None def bind_verdict(self, handler: Callable[[str, str], bool] | None) -> None: @@ -124,19 +131,15 @@ def route_line(self, line: str) -> bool: if notes: self._record_feedback(notes) return False - if self._pending is None or self._pending_kind != expected: - logger.warning( - "%s refused: no pending %s request.", command, expected - ) - else: - if command == "/continue": - self._continue_revision += 1 - self._pending[1].put(command[1:] + (" " + notes if notes else "")) - if notes: - self._record_feedback(notes) - self._pending = None - self._pending_kind = None - logger.info("%s accepted.", command) + request_id = ( + self._pending[0] + if self._pending is not None and self._pending_kind == expected + else "" + ) + try: + self.reply(request_id, command[1:], notes) + except ValueError as exc: + logger.warning("%s refused: %s", command, exc) return True if command in {"/success", "/failure", "/abort"}: verdict = command[1:] @@ -151,27 +154,51 @@ def route_line(self, line: str) -> bool: with self._lock: self._record_feedback(notes) return True - with self._lock: - pending = self._pending - if pending is None or parsed.request_id != pending[0]: - logger.warning( - "No matching operator request; use /operator ." - ) - else: - if self._pending_kind == "verdict" and parsed.answer == "continue": - self._continue_revision += 1 - reply = parsed.answer + (" " + notes if notes else "") - pending[1].put(reply) - self._record_feedback(reply) - self._pending = None - self._pending_kind = None + try: + self.reply(parsed.request_id, parsed.answer, notes) + except ValueError as exc: + logger.warning("%s", exc) return True - @property - def continue_revision(self) -> int: - """Count accepted continue replies, including replies during a model turn.""" + def snapshot(self) -> dict | None: + """Return the current request for the terminal or Dashboard transport.""" + with self._lock: + if self._pending is None: + return None + return { + "id": self._pending[0], + "kind": self._pending_kind, + "prompt": self._pending_prompt, + "choices": self._choices(), + } + + def _choices(self) -> list[str]: + return ( + ["done", "abort"] + if self._pending_kind == "reset" + else ["success", "failure", "continue", "abort"] + ) + + def reply(self, request_id: str, answer: str, notes: str = "") -> None: + """Accept one matching confirmation; reject expired or invalid replies.""" with self._lock: - return self._continue_revision + if self._closed or self._pending is None or request_id != self._pending[0]: + raise ValueError("expired or already answered operator request") + if ( + self._pending_kind is not None and answer not in self._choices() + ) or not isinstance(notes, str): + raise ValueError("invalid operator response") + reply = answer + (" " + notes.strip() if notes.strip() else "") + self._pending[1].put(reply) + if notes: + self._record_feedback(notes.strip()) + self._pending = None + self._pending_kind = None + self._notify_change() + + def _notify_change(self) -> None: + if self._on_change is not None: + self._on_change() def _record_feedback(self, text: str) -> None: self._feedback.append(text) @@ -201,6 +228,7 @@ def close(self) -> None: if self._pending is not None: self._pending[1].put(None) self._pending = None + self._notify_change() @property def pending_kind(self) -> str | None: @@ -219,6 +247,7 @@ def cancel_pending(self) -> None: self._pending[1].put(None) self._pending = None self._pending_kind = None + self._notify_change() def __call__(self, prompt: str, check_cancelled: Callable[[], None]) -> str | None: return self.request(prompt, check_cancelled) @@ -252,6 +281,7 @@ def request( raise RuntimeError("another operator request is pending") self._pending = (request_id, replies) self._pending_kind = kind + self._pending_prompt = prompt shortcut = ( "/done" if kind == "reset" else "/continue" if kind == "verdict" else None ) @@ -262,9 +292,12 @@ def request( else "" ) try: - print( - f"\n{prompt}\nReply: /operator {request_id} {hint}", flush=True - ) + if self._on_change is None: + print( + f"\n{prompt}\nReply: /operator {request_id} {hint}", + flush=True, + ) + self._notify_change() while True: check_cancelled() try: @@ -276,3 +309,4 @@ def request( if self._pending is not None and self._pending[0] == request_id: self._pending = None self._pending_kind = None + self._notify_change() diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index f0b7cc1ad..2d66a1765 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -337,11 +337,6 @@ def cancel_active_and_wait(self) -> None: """Pause admission, cancel queued and active calls, and wait for cleanup.""" self._scheduler.wait(self._scheduler.cancel()) - def wait_active(self) -> None: - """Wait for admitted calls to complete without cancelling them.""" - with self._scheduler.condition: - self._scheduler.condition.wait_for(lambda: not self._scheduler.active) - def resume_calls(self) -> None: """Reopen admission after interrupted calls and their workers have drained.""" self._scheduler.resume() diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index e788efa77..02b39feba 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -25,7 +25,6 @@ from robots.dual_franka.toolkit import DualFrankaToolkit from rpent.dashboard.events import NullDashboardEventSink from rpent.memory import MemoryManager -from rpent.planner.base import PlannerResult class FakeEnv: @@ -571,7 +570,7 @@ def test_direct_success_with_failed_observation_does_not_publish(setup): @pytest.mark.parametrize("robot_name", ["dual_franka", "libero"]) -@pytest.mark.parametrize("verdict", ["success", "failure", "abort"]) +@pytest.mark.parametrize("verdict", ["success", "failure", "abort", "eof"]) @pytest.mark.parametrize("planner_error", [None, "planner transport failed"]) def test_cli_direct_verdict_finalizes_and_merges_only_without_errors( tmp_path, @@ -589,9 +588,12 @@ def test_cli_direct_verdict_finalizes_and_merges_only_without_errors( env = FakeEnv() handlers = {} + command = verdict + verdict = "abort" if command == "eof" else verdict def reader(input_queue, **kwargs): handlers["line"] = kwargs["line_handler"] + handlers["close"] = kwargs["on_close"] input_queue.put("test task") monkeypatch.setattr(cli, "start_interactive_reader", reader) @@ -601,8 +603,12 @@ def solve(self, *, toolkit, input_queue, **kwargs): toolkit._operator_input = lambda *args: "done" call(toolkit, "request_scene_reset", reason="test") call(toolkit, "move_delta", arm="right", delta_xyz=[0.01, 0, 0]) - assert handlers["line"]("/" + verdict) - assert input_queue.cancelled + if command == "eof": + handlers["close"]() + input_queue.put(None) + else: + assert handlers["line"]("/" + verdict) + assert input_queue.cancelled assert input_queue.get() is None return SimpleNamespace( finish_result=None, messages=[], stats={}, error=planner_error @@ -723,8 +729,9 @@ def observe(): np.testing.assert_allclose(env.moves[0][1], [0.01, 0, 0]) +@pytest.mark.parametrize("ending", ["/success checked", None]) def test_attended_confirmation_resumes_same_attempt_over_http( - setup, tmp_path, monkeypatch + setup, tmp_path, monkeypatch, ending ): import asyncio import queue @@ -734,7 +741,8 @@ def test_attended_confirmation_resumes_same_attempt_over_http( from mcp import ClientSession from mcp.client.streamable_http import streamable_http_client - from rpent.cli.attended import _solve_attended + from rpent.planner.codex import CodexPlanner + from rpent.session.input import SessionInputQueue from rpent.tools.human_in_the_loop import HumanInTheLoopInput toolkit, env, replies = setup @@ -743,52 +751,85 @@ def test_attended_confirmation_resumes_same_attempt_over_http( ready, prompted = threading.Event(), threading.Event() monkeypatch.setattr("builtins.print", lambda *a, **kw: prompted.set()) monkeypatch.setattr( - "rpent.cli.attended.logger.info", + "rpent.planner.codex.logger.info", lambda message, *args: ( - ready.set() if "current operator request" in message else None + ready.set() if "Waiting for feedback" in message else None ), ) - inputs = queue.Queue() - calls, requests = [], [] - - async def request(url): - async with httpx.AsyncClient(trust_env=False) as client: - async with streamable_http_client(url, http_client=client) as ( + inputs = SessionInputQueue(queue.Queue()) + clients, prompts = [], [] + endpoint = [] + + async def request(name, args): + async with ( + httpx.AsyncClient(trust_env=False) as client, + streamable_http_client(endpoint[0], http_client=client) as ( read, write, _, - ): - async with ClientSession(read, write) as session: - await session.initialize() - name = "request_scene_reset" - args = {"reason": "test"} - result = await session.call_tool(name, args) - assert not result.isError - - class Planner: - def solve(self, *, toolkit, user_message, mcp_server, **kwargs): - calls.append(mcp_server) - if len(calls) == 1: - worker = threading.Thread( - target=lambda: asyncio.run(request(mcp_server.url)) - ) - requests.append(worker) - worker.start() - assert prompted.wait(2) - return PlannerResult() - assert "SAME attempt" in user_message - assert env.resets == 1 - toolkit.request_direct_verdict("success", "checked") - return PlannerResult() + ), + ClientSession(read, write) as session, + ): + await session.initialize() + result = await session.call_tool(name, args) + assert not result.isError + + class Client: + closed = False + + def __init__(self, config): + self.interrupted = asyncio.Event() + clients.append(self) + + async def thread_start(self, **kwargs): + return self + + async def turn(self, text, **kwargs): + prompts.append(text) + return self + + async def stream(self): + if len(prompts) == 1: + await request("request_scene_reset", {"reason": "test"}) + else: + assert env.resets == 1 + await request("move_delta", {"arm": "right", "delta_xyz": [0, 0, 0.01]}) + if ending is None: + inputs.put(None) + else: + assert broker.route_line(ending) + await self.interrupted.wait() + yield { + "method": "turn/completed", + "payload": {"turn": {"status": "completed"}}, + } + + async def interrupt(self): + self.interrupted.set() + + async def close(self): + self.closed = True + + def verdict(value, notes): + accepted = toolkit.request_direct_verdict(value, notes) + if accepted: + inputs.cancel() + return accepted + + broker.bind_verdict(verdict) + monkeypatch.setattr("rpent.planner.codex.openai_codex.AsyncCodex", Client) + planner = CodexPlanner( + output_dir=str(tmp_path), + repo_root=tmp_path, + timeout_s=10, + dashboard_events=NullDashboardEventSink(), + ) + monkeypatch.setattr(planner, "_build_config", lambda url: endpoint.append(url)) results = [] runner = threading.Thread( target=lambda: results.append( - _solve_attended( - Planner(), - operator_input=broker, - state_output_dir=tmp_path / "session", - keep_mcp_alive=True, + planner.solve( toolkit=toolkit, input_queue=inputs, system_prompt="test", @@ -799,17 +840,16 @@ def solve(self, *, toolkit, user_message, mcp_server, **kwargs): ) runner.start() try: - assert ready.wait(2) + assert prompted.wait(3) assert env.resets == 0 assert broker.route_line("/done") + assert ready.wait(3) + inputs.put("continue with this object") runner.join(5) - for worker in requests: - worker.join(2) - assert not worker.is_alive() assert not runner.is_alive() and results[0].error is None - assert len(calls) == 2 and calls[0] is calls[1] - assert calls[0]._thread is None - assert env.resets == 1 and not env.moves + assert len(clients) == 1 and clients[0].closed + assert prompts == ["test\n\ntest task", "continue with this object"] + assert env.resets == 1 and len(env.moves) == 1 finally: inputs.put(None) broker.close() diff --git a/tests/unit_tests/rpent/planner/test_codex_contracts.py b/tests/unit_tests/rpent/planner/test_codex_contracts.py index 1ccaf471c..6fbbebbd8 100644 --- a/tests/unit_tests/rpent/planner/test_codex_contracts.py +++ b/tests/unit_tests/rpent/planner/test_codex_contracts.py @@ -620,7 +620,7 @@ def interrupt(self): def test_dashboard_interrupts_at_response_budget_and_closes( interrupt_error, completed_status, expected_error ): - from rpent.planner.codex import _CodexDashboardSession, _Recorder + from rpent.planner.codex import _CodexSession, _Recorder async def run(): class Turn: @@ -671,7 +671,7 @@ def emit_event(event): events.append(event) recorder.observe(event) - session = _CodexDashboardSession( + session = _CodexSession( config=None, thread_options={}, turn_options={}, @@ -1227,31 +1227,3 @@ def test_retry_error_is_visible_without_poisoning_successful_turn(): ) assert "Retries exhausted" in recorder.error assert "Model connection failed" in str(sink.events[-1]) - - -def test_terminal_eof_interrupts_codex(tmp_path, monkeypatch): - install_fake_backend(monkeypatch) - interrupted = threading.Event() - - def interrupt(turn): - turn.interrupt_calls += 1 - interrupted.set() - - def stream(turn): - assert interrupted.wait(2) - yield from turn.events - - monkeypatch.setattr(FakeTurn, "interrupt", interrupt) - monkeypatch.setattr(FakeTurn, "stream", stream) - inputs = queue.Queue() - inputs.put(None) - result = make_planner(tmp_path, RecordingSink()).solve( - system_prompt="test", - user_message="task", - toolkit=FakeToolkit(), - max_turns=10, - input_queue=inputs, - ) - assert result.error is None - assert FakeCodex.instances[-1].thread.fake_turn.interrupt_calls == 1 - assert inputs.empty() From 1b067e3c13c8d36a34195f4123357fb07a9147e6 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 01:53:26 +0800 Subject: [PATCH 12/18] refactor(cli): share operator outcome finalization --- rpent/cli/attended.py | 14 +++++++++++--- rpent/cli/dashboard.py | 27 +++++++++++---------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/rpent/cli/attended.py b/rpent/cli/attended.py index 378f04c8e..ec6c9a520 100644 --- a/rpent/cli/attended.py +++ b/rpent/cli/attended.py @@ -25,9 +25,17 @@ def _finalize_operator_verdict( - result, *, toolkit, memory, recipe_tag, output_dir, state_output_dir, session_number + result, + *, + toolkit, + memory, + recipe_tag, + output_dir, + state_output_dir, + session_number, + archive=True, ): - """Archive the interrupted attempt and prepare its next-session handoff.""" + """Finalize the interrupted attempt, optionally archiving exploration evidence.""" finish = toolkit.finalize_direct_verdict() error = result.error if error: @@ -35,7 +43,7 @@ def _finalize_operator_verdict( result.stats["planner_error_at_operator_verdict"] = error verdict = finish["operator_verdict"] handoff = "" - if verdict in {"success", "failure"}: + if archive and verdict in {"success", "failure"}: evidence = record_operator_outcome( memory_root=memory.root, recipe_tag=recipe_tag, diff --git a/rpent/cli/dashboard.py b/rpent/cli/dashboard.py index eda9f0982..6277f2f78 100644 --- a/rpent/cli/dashboard.py +++ b/rpent/cli/dashboard.py @@ -27,13 +27,13 @@ from pathlib import Path from typing import TYPE_CHECKING, Any +from rpent.cli.attended import _finalize_operator_verdict from rpent.cli.main import ( _handoff_message, _serialize_messages, ) from rpent.dashboard.events import RunStartedEvent from rpent.evaluation import RunFinalizationContext -from rpent.evaluation.result import record_operator_outcome from rpent.memory import MemoryManager from rpent.planner.base import build_planner from rpent.robots import get_toolkit @@ -332,21 +332,16 @@ def _run_dashboard_task( stats = result.stats agent_error = result.error if human_interactive_run and toolkit.direct_verdict_requested: - finish_result = toolkit.finalize_direct_verdict() - if task_args.explore and finish_result["operator_verdict"] in { - "success", - "failure", - }: - record_operator_outcome( - memory_root=memory_manager.root, - recipe_tag=recipe_tag, - run_name=output_dir.name, - session_number=session_number, - state_output_dir=state_output_dir, - verdict=finish_result, - messages=_serialize_messages(result.messages), - planner_error=agent_error, - ) + finish_result, _, _ = _finalize_operator_verdict( + result, + toolkit=toolkit, + memory=memory_manager, + recipe_tag=recipe_tag, + output_dir=output_dir, + state_output_dir=state_output_dir, + session_number=session_number, + archive=task_args.explore, + ) if robot_spec.supports_exploration: solved = bool(toolkit.solved()) if solved: From aec6c475293f297e12f0f5b88b9c09dfef33eaec Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 16:08:47 +0800 Subject: [PATCH 13/18] fix(exploration): serialize observations and retain stable outcome evidence --- robots/dual_franka/toolkit.py | 5 ++ rpent/cli/attended.py | 39 +++++---- rpent/cli/dashboard.py | 5 +- rpent/cli/main.py | 5 +- rpent/dashboard/state.py | 5 +- rpent/evaluation/result.py | 13 +-- rpent/tools/toolkit.py | 12 ++- .../robots/dual_franka/test_exploration.py | 80 +++++++++++++++++++ .../rpent/evaluation/test_result.py | 29 +++++++ 9 files changed, 156 insertions(+), 37 deletions(-) diff --git a/robots/dual_franka/toolkit.py b/robots/dual_franka/toolkit.py index 639606228..8546f628d 100644 --- a/robots/dual_franka/toolkit.py +++ b/robots/dual_franka/toolkit.py @@ -227,6 +227,11 @@ def _guard_motion(self, inner, **kwargs) -> ToolResult: self._clear_verdict() return inner(**kwargs) + def _is_readonly_call(self, tool: Tool, kwargs: dict[str, Any]) -> bool: + if tool.name == "view_env_state": + return kwargs["step"] >= 0 + return tool.readonly and tool.name != "request_operator_verdict" + def _view_env_state(self, step: int = -1) -> ToolResult: if step >= 0: return dual_franka_tools.view_env_state(step, state=self._state) diff --git a/rpent/cli/attended.py b/rpent/cli/attended.py index ec6c9a520..6194c90bb 100644 --- a/rpent/cli/attended.py +++ b/rpent/cli/attended.py @@ -14,8 +14,12 @@ """CLI orchestration for one operator-attended attempt.""" +from __future__ import annotations + import shutil +from collections.abc import Callable from pathlib import Path +from typing import TYPE_CHECKING, Any from rpent.evaluation.result import record_operator_outcome from rpent.session.transcript import _serialize_messages @@ -23,20 +27,21 @@ logger = get_logger("agent") +if TYPE_CHECKING: + from rpent.memory import MemoryManager + from rpent.planner.base import PlannerResult + def _finalize_operator_verdict( - result, + result: PlannerResult, *, - toolkit, - memory, - recipe_tag, - output_dir, - state_output_dir, - session_number, - archive=True, -): - """Finalize the interrupted attempt, optionally archiving exploration evidence.""" - finish = toolkit.finalize_direct_verdict() + finalize_verdict: Callable[[], dict[str, Any]], + state_output_dir: Path, + session_number: int, + archive: bool = True, +) -> tuple[dict[str, Any], str | None, str]: + """Return the finalized outcome, remaining planner error, and next-session handoff.""" + finish = finalize_verdict() error = result.error if error: logger.info("Planner stopped after operator verdict: %s", error) @@ -45,9 +50,6 @@ def _finalize_operator_verdict( handoff = "" if archive and verdict in {"success", "failure"}: evidence = record_operator_outcome( - memory_root=memory.root, - recipe_tag=recipe_tag, - run_name=Path(output_dir).name, session_number=session_number, state_output_dir=state_output_dir, verdict=finish, @@ -68,8 +70,13 @@ def _finalize_operator_verdict( def _publish_success_recipe( - memory, *, recipe_tag, output_dir, state_output_dir, auto_merge -): + memory: MemoryManager, + *, + recipe_tag: str, + output_dir: Path, + state_output_dir: Path, + auto_merge: bool, +) -> None: """Preserve successful-session artifacts before another session overwrites them.""" for name in (f"{recipe_tag}.json", f"{recipe_tag}_recipe.jsonl"): shutil.copy2(Path(output_dir) / name, Path(state_output_dir) / name) diff --git a/rpent/cli/dashboard.py b/rpent/cli/dashboard.py index 6277f2f78..0d680b31a 100644 --- a/rpent/cli/dashboard.py +++ b/rpent/cli/dashboard.py @@ -334,10 +334,7 @@ def _run_dashboard_task( if human_interactive_run and toolkit.direct_verdict_requested: finish_result, _, _ = _finalize_operator_verdict( result, - toolkit=toolkit, - memory=memory_manager, - recipe_tag=recipe_tag, - output_dir=output_dir, + finalize_verdict=toolkit.finalize_direct_verdict, state_output_dir=state_output_dir, session_number=session_number, archive=task_args.explore, diff --git a/rpent/cli/main.py b/rpent/cli/main.py index 7c1372a78..fc1ce0797 100644 --- a/rpent/cli/main.py +++ b/rpent/cli/main.py @@ -620,10 +620,7 @@ def accept_verdict( finish_result, agent_error, verdict_handoff = ( _finalize_operator_verdict( result, - toolkit=toolkit, - memory=memory_manager, - recipe_tag=recipe_tag, - output_dir=output_dir, + finalize_verdict=toolkit.finalize_direct_verdict, state_output_dir=state_output_dir, session_number=session_number, ) diff --git a/rpent/dashboard/state.py b/rpent/dashboard/state.py index 77875999b..d9a43a171 100644 --- a/rpent/dashboard/state.py +++ b/rpent/dashboard/state.py @@ -51,6 +51,7 @@ if TYPE_CHECKING: from rpent.session import EnvState, StepRecord + from rpent.tools.human_in_the_loop import HumanInTheLoopInput from rpent.tools.toolkit import Toolkit, ToolResult RUNTIME_STATUSES = {"pending", "starting", "ready", "failed"} @@ -236,8 +237,8 @@ def __init__( self._manual_results: list[str] = [] self._explore_paused = False self._exploration_message: DashboardMessage | None = None - self.operator = None - self.operator_verdict = None + self.operator: HumanInTheLoopInput | None = None + self.operator_verdict: str | None = None def operator_changed(self): with self._condition: diff --git a/rpent/evaluation/result.py b/rpent/evaluation/result.py index 7d0c88191..595837d2d 100644 --- a/rpent/evaluation/result.py +++ b/rpent/evaluation/result.py @@ -50,9 +50,6 @@ class RunFinalizationContext: def record_operator_outcome( *, - memory_root: Path, - recipe_tag: str, - run_name: str, session_number: int, state_output_dir: str | Path, verdict: dict[str, Any], @@ -62,11 +59,9 @@ def record_operator_outcome( """Archive an operator outcome without overwriting a previous attempt. Args: - memory_root: Root of the run's memory store. - recipe_tag: Cell whose inbox receives the evidence. - run_name: Run identifier used to distinguish evidence files. session_number: Attempt number within the run. - state_output_dir: Directory containing the attempt's observations. + state_output_dir: Unique attempt directory containing its observations. + Evidence stays here when memory drafts are merged. verdict: Operator-confirmed result and notes. messages: Serializable planner transcript. planner_error: Error recorded before operator finalization, if any. @@ -74,9 +69,7 @@ def record_operator_outcome( Returns: Path to the archived evidence JSON. """ - outcome = verdict["operator_verdict"] - path = memory_root / "_internal" / "inbox" / recipe_tag / "wip" - path /= f"{run_name}-session-{session_number:03d}-{outcome}.json" + path = Path(state_output_dir).resolve() / "operator_outcome.json" path.parent.mkdir(parents=True, exist_ok=True) with path.open("x") as file: json.dump( diff --git a/rpent/tools/toolkit.py b/rpent/tools/toolkit.py index 2d66a1765..0f6f7a922 100644 --- a/rpent/tools/toolkit.py +++ b/rpent/tools/toolkit.py @@ -235,7 +235,9 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: } try: - operation = self._scheduler.acquire(readonly=tool.readonly) + operation = self._scheduler.acquire( + readonly=self._is_readonly_call(tool, kwargs) + ) except ToolCancelled as exc: return ToolResult( data={"error": str(exc), "code": "tool_cancelled", "interrupted": True} @@ -310,6 +312,14 @@ def execute_tool(self, name: str, input_dict: dict[str, Any]) -> ToolResult: finally: self._scheduler.release(operation) + def _is_readonly_call(self, tool: Tool, kwargs: dict[str, Any]) -> bool: + """Choose shared admission independently of automatic state capture. + + Self-recording observation tools can require exclusive admission while + retaining their declaration's suppression of automatic capture. + """ + return tool.readonly + def _publish_step(self, record: StepRecord) -> None: """Publish one recorded environment step to the dashboard sink.""" self._dashboard_events.emit( diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index 02b39feba..708d205fe 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -729,6 +729,86 @@ def observe(): np.testing.assert_allclose(env.moves[0][1], [0.01, 0, 0]) +@pytest.mark.parametrize("second_tool", ["view_env_state", "request_operator_verdict"]) +def test_fresh_observation_and_verdict_serialize_state_writes( + setup, monkeypatch, second_tool +): + import threading + from concurrent.futures import ThreadPoolExecutor + + t, env, replies = setup + reset(t, replies) + initial_step = t.state.latest_step + entered, release = threading.Event(), threading.Event() + save = t.state.save + + def hold_first_image(name, value, **kwargs): + if name.endswith(".png") and not entered.is_set(): + entered.set() + assert release.wait(5) + return save(name, value, **kwargs) + + monkeypatch.setattr(t.state, "save", hold_first_image) + replies.append("success") + with ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit(t.execute_tool, "view_env_state", {}) + try: + assert entered.wait(5) + second = pool.submit(t.execute_tool, second_tool, {}) + with t._scheduler.condition: + assert t._scheduler.condition.wait_for(lambda: t._scheduler.pending, 5) + finally: + release.set() + assert not first.result(5).is_error + assert not second.result(5).is_error + assert t.state.latest_step == initial_step + 2 and t._scene_ready + assert not call(t, "move_delta", arm="left", delta_xyz=[0.01, 0, 0]).get("error") + assert len(env.moves) == 1 and env.resets == 1 + + +def test_operator_evidence_remains_readable_after_success_memory_merge(setup, tmp_path): + from rpent.cli.attended import _finalize_operator_verdict, _publish_success_recipe + from rpent.planner.base import PlannerResult + + t, _, replies = setup + reset(t, replies) + call(t, "move_delta", arm="left", delta_xyz=[0.01, 0, 0]) + tag = "dual_franka_t0" + note = t.memory.authorize_write( + t.memory.root / "_internal" / "inbox" / tag / "strategy.md" + ) + note.parent.mkdir(parents=True, exist_ok=True) + note.write_text( + "---\nscope: global\nkind: strategy\ntitle: example\napplies_when: example\n" + "confidence: single-shot\nevidence:\n cells: [dual_franka_t0]\n---\nObserved example.\n" + ) + assert t.request_direct_verdict("success") + state_dir = tmp_path / "sessions" / "session_001" + finish, error, handoff = _finalize_operator_verdict( + PlannerResult(), + finalize_verdict=t.finalize_direct_verdict, + state_output_dir=state_dir, + session_number=1, + ) + evidence = state_dir / "operator_outcome.json" + assert finish["status"] == "success" and error is None + assert str(evidence) in handoff + original = evidence.read_bytes() + t.write_recipe(tag) + _publish_success_recipe( + t.memory, + recipe_tag=tag, + output_dir=tmp_path, + state_output_dir=state_dir, + auto_merge=True, + ) + assert (t.memory.root / "global" / "strategy.md").is_file() + next_memory = MemoryManager( + t.memory.root, memory_access="inbox_write", inbox_cell_tag=tag + ) + assert next_memory.authorize_read(evidence).read_bytes() == original + + @pytest.mark.parametrize("ending", ["/success checked", None]) def test_attended_confirmation_resumes_same_attempt_over_http( setup, tmp_path, monkeypatch, ending diff --git a/tests/unit_tests/rpent/evaluation/test_result.py b/tests/unit_tests/rpent/evaluation/test_result.py index f43b031fa..4c37b8a72 100644 --- a/tests/unit_tests/rpent/evaluation/test_result.py +++ b/tests/unit_tests/rpent/evaluation/test_result.py @@ -23,6 +23,35 @@ from rpent.evaluation import write_json_atomic +def test_operator_outcomes_are_unique_across_dashboard_runs(tmp_path: Path) -> None: + from robots.dual_franka.robot_spec import DUAL_FRANKA_DASHBOARD_SPEC + from rpent.dashboard.state import DashboardState + from rpent.evaluation.result import record_operator_outcome + + paths = [] + for run in ("dashboard_a", "dashboard_b"): + state = DashboardState( + output_dir=tmp_path / run, dashboard_spec=DUAL_FRANKA_DASHBOARD_SPEC + ) + state.shared_services_ready() + state.request_task({"task_id": 3}) + task = state.wait_for_task(0) + kwargs = { + "state_output_dir": task.output_dir / "sessions" / "session_001", + "session_number": 1, + "verdict": {"operator_verdict": "failure"}, + "messages": [], + "planner_error": None, + } + evidence = record_operator_outcome(**kwargs) + before = evidence.read_bytes() + with pytest.raises(FileExistsError): + record_operator_outcome(**kwargs) + assert evidence.read_bytes() == before + paths.append(evidence) + assert paths[0] != paths[1] and all(path.is_file() for path in paths) + + def test_write_json_atomic_replaces_complete_record(tmp_path: Path) -> None: destination = tmp_path / "result.json" destination.write_text('{"status": "old"}\n', encoding="utf-8") From 2b966762891fa1005336ab8c858f021198b984fa Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 16:32:52 +0800 Subject: [PATCH 14/18] test(interaction): reuse attended workflow fixtures --- .../unit_tests/robots/dual_franka/conftest.py | 83 ++++++++++ .../dual_franka/test_dashboard_operator.py | 41 +---- .../dual_franka/test_dual_franka_extension.py | 38 ++--- .../robots/dual_franka/test_exploration.py | 151 +++--------------- tests/unit_tests/rpent/conftest.py | 20 +++ .../rpent/dashboard/test_state_contracts.py | 5 +- .../rpent/planner/test_claude_contracts.py | 7 +- 7 files changed, 142 insertions(+), 203 deletions(-) create mode 100644 tests/unit_tests/rpent/conftest.py diff --git a/tests/unit_tests/robots/dual_franka/conftest.py b/tests/unit_tests/robots/dual_franka/conftest.py index 75aa8e9e9..d00a2fc1c 100644 --- a/tests/unit_tests/robots/dual_franka/conftest.py +++ b/tests/unit_tests/robots/dual_franka/conftest.py @@ -15,8 +15,11 @@ """Shared fixtures for offline dual-Franka tests.""" from collections.abc import Iterator +from dataclasses import replace from pathlib import Path +from types import SimpleNamespace +import numpy as np import pytest import yaml @@ -24,6 +27,86 @@ from robots.franka.runtime_config import set_robot_config_path +class FakeEnv: + def __init__(self): + self.meta = {} + self.resets = 0 + self.moves = [] + self.reset_result = {"ok": True} + self.fail_observation = False + + def reset(self): + self.resets += 1 + return self.reset_result + + def get_observation(self): + if self.fail_observation: + raise RuntimeError("camera offline") + return { + "main_images": np.full((8, 8, 3), self.resets, dtype=np.uint8), + "extra_view_images": np.zeros((2, 8, 8, 3), dtype=np.uint8), + "main_depths": np.ones((8, 8), dtype=np.float32), + "extra_view_depths": np.ones((2, 8, 8), dtype=np.float32), + "d455_images": np.zeros((8, 8, 3), dtype=np.uint8), + "d455_depths": np.ones((8, 8), dtype=np.float32), + } + + def get_robot_state(self): + return { + "left_arm": {"tcp_pose": [0.5, 0, 0.5, 0, 0, 0, 1]}, + "right_arm": {"tcp_pose": [0.5, 0, 0.5, 0, 0, 0, 1]}, + } + + def get_camera_meta(self): + return {"cameras": {}} + + def move_delta(self, arm, delta): + self.moves.append((arm, list(delta))) + return {"ok": True} + + +@pytest.fixture +def fake_env(): + return FakeEnv() + + +@pytest.fixture +def dual_franka_run(tmp_path, dual_franka_robot_config, fake_env): + from robots.dual_franka import robot_spec + from rpent.cli import main + + calls = [] + + def init_runtime(*args): + calls.append(args) + return [], {"env": fake_env, "model": None, "task_description": "test"} + + run = SimpleNamespace( + env=fake_env, + spec=replace(robot_spec.get_robot_spec(), init_runtime=init_runtime), + runtime_calls=calls, + argv=[ + "rpent", + "--robot", + "dual_franka", + "--planner", + "codex", + "--explore", + "--auto-merge-memory", + "--robot-config", + str(dual_franka_robot_config), + "--memory-dir", + str(tmp_path / "memory"), + "--output-dir", + str(tmp_path / "run"), + ], + ) + parser = main._build_argparser() + run.spec.add_cli_args(parser, use_dashboard=False) + run.args = parser.parse_args(run.argv[1:]) + return run + + @pytest.fixture def dual_franka_robot_config(tmp_path: Path) -> Iterator[Path]: """Return the packaged config with calibration paths redirected to fixtures.""" diff --git a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py index 3b157e145..1fec241a3 100644 --- a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py +++ b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py @@ -16,51 +16,24 @@ import time from concurrent.futures import ThreadPoolExecutor -from dataclasses import replace from types import SimpleNamespace from fastapi.testclient import TestClient -from robots.dual_franka import robot_spec -from rpent.cli import dashboard, main +from rpent.cli import dashboard from rpent.dashboard.server import DashboardServer from rpent.dashboard.state import DashboardState from rpent.planner.base import PlannerResult -from tests.unit_tests.robots.dual_franka.test_exploration import FakeEnv def test_web_operator_confirms_reset_and_completes_task( - tmp_path, monkeypatch, dual_franka_robot_config + tmp_path, monkeypatch, dual_franka_run ): - parser = main._build_argparser() - robot_spec._add_cli_args(parser, use_dashboard=True) - args = parser.parse_args( - [ - "--robot", - "dual_franka", - "--dashboard", - "--explore", - "--task-id", - "3", - "--planner", - "codex", - "--auto-merge-memory", - "--robot-config", - str(dual_franka_robot_config), - "--memory-dir", - str(tmp_path / "memory"), - "--output-dir", - str(tmp_path / "run"), - ] - ) - env = FakeEnv() - spec = replace( - robot_spec.get_robot_spec(), - init_runtime=lambda *a: ( - [], - {"env": env, "model": None, "task_description": "test"}, - ), - ) + spec = dual_franka_run.spec + args = dual_franka_run.args + args.dashboard = True + args.task_id = 3 + env = dual_franka_run.env state = DashboardState(output_dir=tmp_path, dashboard_spec=spec.dashboard) state.shared_services_ready() state.request_task({"task_id": 3}) diff --git a/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py b/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py index 72ffa752b..2c63f0257 100644 --- a/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py +++ b/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py @@ -19,10 +19,8 @@ from pathlib import Path from robots.dual_franka import get_robot_spec -from robots.dual_franka.prompt_bundle import system_prompt, user_prompt from robots.dual_franka.runtime_config import load_runtime_config from robots.dual_franka.tasks import CLEAN_DESK_VLA_PROMPT, DUAL_FRANKA_TASKS -from rpent.prompt.utils import format_prompt from rpent.robots.base import enumerate_robots from rpent.robots.base import get_robot_spec as resolve_robot_spec @@ -120,34 +118,16 @@ def test_metal_basket_task_registers_non_sorting_prompt(): assert "green object before the blue object" in constraints -def test_dual_franka_exploration_prompt_is_opt_in(): - eval_vars = {"mode": "eval"} - explore_vars = { - "mode": "explore", - "task_id": 4, - "session_number": 1, - "session_max": 1, - "recipe_tag": "dual_franka_t4", - "memory_dir": "/tmp/memory", - "task_name": "offline", - "instruction": "test instruction", - "setup": "test setup", - "success_criteria": "test success", - "constraints": "1. test constraint", - "output_dir": "/tmp/run", - "memory_inbox": "/tmp/memory/_internal/inbox/dual_franka_t4", - } - eval_prompt = format_prompt(system_prompt(eval_vars), variables=eval_vars) - explore_prompt = format_prompt( - system_prompt(explore_vars), - variables=explore_vars, - ) - explore_user_prompt = format_prompt( - user_prompt(explore_vars), - variables=explore_vars, - ) +def test_dual_franka_exploration_prompt_is_opt_in(dual_franka_run): + spec = dual_franka_run.spec + dual_franka_run.args.task_id = 4 + config = spec.parse_config(dual_franka_run.args) + variables = {**config.prompt_vars, "output_dir": config.output_dir} + eval_prompt = spec.prompts.render("system", variables={**variables, "mode": "eval"}) + explore_prompt = spec.prompts.render("system", variables=variables) + explore_user_prompt = spec.prompts.render("user", variables=variables) assert "request_scene_reset" not in eval_prompt assert "request_scene_reset" in explore_prompt assert "request_operator_verdict" in explore_prompt - assert "/tmp/memory/_internal/inbox/dual_franka_t4" in explore_user_prompt + assert variables["memory_inbox"] in explore_user_prompt diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index 708d205fe..fcb04822b 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -14,7 +14,6 @@ """Offline attended exploration: real toolkit, logger and memory, fake hardware.""" -import argparse import json from pathlib import Path @@ -27,46 +26,8 @@ from rpent.memory import MemoryManager -class FakeEnv: - def __init__(self): - self.meta = {} - self.resets = 0 - self.moves = [] - self.reset_result = {"ok": True} - self.fail_observation = False - - def reset(self): - self.resets += 1 - return self.reset_result - - def get_observation(self): - if self.fail_observation: - raise RuntimeError("camera offline") - return { - "main_images": np.full((8, 8, 3), self.resets, dtype=np.uint8), - "extra_view_images": np.zeros((2, 8, 8, 3), dtype=np.uint8), - "main_depths": np.ones((8, 8), dtype=np.float32), - "extra_view_depths": np.ones((2, 8, 8), dtype=np.float32), - "d455_images": np.zeros((8, 8, 3), dtype=np.uint8), - "d455_depths": np.ones((8, 8), dtype=np.float32), - } - - def get_robot_state(self): - return { - "left_arm": {"tcp_pose": [0.5, 0, 0.5, 0, 0, 0, 1]}, - "right_arm": {"tcp_pose": [0.5, 0, 0.5, 0, 0, 0, 1]}, - } - - def get_camera_meta(self): - return {"cameras": {}} - - def move_delta(self, arm, delta): - self.moves.append((arm, list(delta))) - return {"ok": True} - - @pytest.fixture -def setup(tmp_path, monkeypatch, dual_franka_robot_config): +def setup(tmp_path, monkeypatch, dual_franka_robot_config, fake_env): from rpent.utils import logging monkeypatch.setattr(logging, "_output_dir", tmp_path) @@ -74,7 +35,7 @@ def setup(tmp_path, monkeypatch, dual_franka_robot_config): from robots.franka.runtime_config import set_robot_config_path set_robot_config_path(dual_franka_robot_config) - env = FakeEnv() + env = fake_env replies = [] toolkit = DualFrankaToolkit( runtime_kwargs={"env": env, "model": None, "task_description": "test"}, @@ -190,28 +151,9 @@ def test_budget_and_old_perception_boundary(setup): assert env.resets == 2 -def test_explore_prompt_and_factory_use_local_layered_memory( - tmp_path, dual_franka_robot_config -): - parser = argparse.ArgumentParser() - parser.add_argument("--explore", action="store_true") - parser.add_argument("--memory-dir") - parser.add_argument("--memory-profile", default=None) - parser.add_argument("--output-dir") - robot_spec.get_robot_spec().add_cli_args(parser, False) - args = parser.parse_args( - [ - "--explore", - "--memory-dir", - str(tmp_path / "memory"), - "--output-dir", - str(tmp_path), - "--robot-config", - str(dual_franka_robot_config), - ] - ) - config = robot_spec.get_robot_spec().parse_config(args) - prompt = robot_spec.get_robot_spec().prompts.render( +def test_explore_prompt_and_factory_use_local_layered_memory(tmp_path, dual_franka_run): + config = dual_franka_run.spec.parse_config(dual_franka_run.args) + prompt = dual_franka_run.spec.prompts.render( "system", variables={**config.prompt_vars, "output_dir": tmp_path} ) assert "request_scene_reset" in prompt and "request_operator_verdict" in prompt @@ -222,7 +164,11 @@ def test_explore_prompt_and_factory_use_local_layered_memory( ) assert "{{" not in prompt and "libero_terminated" not in prompt t = robot_spec.get_toolkit( - runtime_kwargs={"env": FakeEnv(), "model": None, "task_description": "test"}, + runtime_kwargs={ + "env": dual_franka_run.env, + "model": None, + "task_description": "test", + }, dashboard_events=NullDashboardEventSink(), config=config, mode="exploration", @@ -286,17 +232,15 @@ def test_successful_memory_pair_uses_existing_merge_and_index(setup, tmp_path): @pytest.mark.parametrize("interactive", [False, True]) def test_cli_two_sessions_operator_feedback_and_memory_pipeline( - tmp_path, monkeypatch, interactive, dual_franka_robot_config + tmp_path, monkeypatch, interactive, dual_franka_run ): import sys - from dataclasses import replace from types import SimpleNamespace from rpent.cli import main as cli from rpent.tools.human_in_the_loop import HumanInTheLoopInput - env = FakeEnv() - runtimes = [] + env = dual_franka_run.env planners = [] replies = iter(["done", "failure dropped", "done", "success lifted"]) @@ -315,10 +259,6 @@ def __init__(self, *args, **kwargs): def solve(self, *, toolkit, system_prompt, user_message, **kwargs): assert "scope: global" in system_prompt - if self.number == 2: - assert "not automatically reset" in user_message - assert "task_name:" in user_message - assert "Original operator task instruction:" in user_message assert not toolkit._scene_ready assert "error" not in call( toolkit, "request_scene_reset", reason="new attempt" @@ -354,16 +294,7 @@ def solve(self, *, toolkit, system_prompt, user_message, **kwargs): finish_result=finish, messages=[], stats={}, error=None ) - def init_runtime(*args): - runtimes.append(args) - return [], { - "env": env, - "model": None, - "task_description": "test", - } - - spec = replace(robot_spec.get_robot_spec(), init_runtime=init_runtime) - monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) + monkeypatch.setattr(cli, "get_robot_spec", lambda name: dual_franka_run.spec) monkeypatch.setattr(cli, "build_planner", Planner) monkeypatch.setattr( cli, @@ -376,28 +307,17 @@ def init_runtime(*args): sys, "argv", [ - "rpent", - "--robot", - "dual_franka", - "--planner", - "codex", - "--explore", + *dual_franka_run.argv, "--explore-sessions", "2", "--explore-attempts-per-session", "1", - "--output-dir", - str(tmp_path / "run"), - "--memory-dir", - str(tmp_path / "memory"), - "--robot-config", - str(dual_franka_robot_config), - "--auto-merge-memory", *(["--interactive"] if interactive else []), ], ) assert cli.main() == 0 - assert len(planners) == 2 and len(runtimes) == 1 and env.resets == 2 + assert len(planners) == 2 and len(dual_franka_run.runtime_calls) == 1 + assert env.resets == 2 traces = [ json.loads((tmp_path / f"run/sessions/session_{i:03d}/states.json").read_text()) for i in (1, 2) @@ -575,18 +495,16 @@ def test_direct_success_with_failed_observation_does_not_publish(setup): def test_cli_direct_verdict_finalizes_and_merges_only_without_errors( tmp_path, monkeypatch, - dual_franka_robot_config, + dual_franka_run, verdict, planner_error, robot_name, ): import sys - from dataclasses import replace from types import SimpleNamespace from rpent.cli import main as cli - env = FakeEnv() handlers = {} command = verdict verdict = "abort" if command == "eof" else verdict @@ -614,18 +532,7 @@ def solve(self, *, toolkit, input_queue, **kwargs): finish_result=None, messages=[], stats={}, error=planner_error ) - spec = replace( - robot_spec.get_robot_spec(), - init_runtime=lambda *args: ( - [], - { - "env": env, - "model": None, - "task_description": "test", - }, - ), - ) - monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) + monkeypatch.setattr(cli, "get_robot_spec", lambda name: dual_franka_run.spec) original_toolkit_factory = cli.get_toolkit monkeypatch.setattr( cli, @@ -634,26 +541,8 @@ def solve(self, *, toolkit, input_queue, **kwargs): ) monkeypatch.setattr(cli, "build_planner", lambda *args, **kwargs: Planner()) monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True)) - monkeypatch.setattr( - sys, - "argv", - [ - "rpent", - "--robot", - robot_name, - "--planner", - "codex", - "--explore", - "--interactive", - "--auto-merge-memory", - "--output-dir", - str(tmp_path / "run"), - "--memory-dir", - str(tmp_path / "memory"), - "--robot-config", - str(dual_franka_robot_config), - ], - ) + dual_franka_run.argv[dual_franka_run.argv.index("--robot") + 1] = robot_name + monkeypatch.setattr(sys, "argv", [*dual_franka_run.argv, "--interactive"]) assert cli.main() == (1 if planner_error and verdict != "failure" else 0) assert (tmp_path / "memory/task-specific/dual_franka_t0.json").is_file() == ( verdict == "success" and planner_error is None diff --git a/tests/unit_tests/rpent/conftest.py b/tests/unit_tests/rpent/conftest.py new file mode 100644 index 000000000..57e83c41c --- /dev/null +++ b/tests/unit_tests/rpent/conftest.py @@ -0,0 +1,20 @@ +# Copyright 2026 The RPent Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture +def tool_template_output_dir(tmp_path, monkeypatch): + monkeypatch.setattr("rpent.utils.templates.get_output_dir", lambda: tmp_path) diff --git a/tests/unit_tests/rpent/dashboard/test_state_contracts.py b/tests/unit_tests/rpent/dashboard/test_state_contracts.py index 39de8f3d4..2aa04b2a0 100644 --- a/tests/unit_tests/rpent/dashboard/test_state_contracts.py +++ b/tests/unit_tests/rpent/dashboard/test_state_contracts.py @@ -38,10 +38,7 @@ from rpent.session import EnvState from rpent.tools import Toolkit, ToolResult, tool - -@pytest.fixture(autouse=True) -def tool_template_output_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("rpent.utils.templates.get_output_dir", lambda: tmp_path) +pytestmark = pytest.mark.usefixtures("tool_template_output_dir") DASHBOARD_SPEC: DashboardSpec = { diff --git a/tests/unit_tests/rpent/planner/test_claude_contracts.py b/tests/unit_tests/rpent/planner/test_claude_contracts.py index 845d489e8..b060d6bc6 100644 --- a/tests/unit_tests/rpent/planner/test_claude_contracts.py +++ b/tests/unit_tests/rpent/planner/test_claude_contracts.py @@ -34,6 +34,8 @@ from rpent.tools import ToolResult, tool from rpent.tools.common import CommonTools +pytestmark = pytest.mark.usefixtures("tool_template_output_dir") + class RecordingSink: def __init__(self) -> None: @@ -583,8 +585,3 @@ async def close(self): await asyncio.gather(running, return_exceptions=True) asyncio.run(scenario()) - - -@pytest.fixture(autouse=True) -def tool_template_output_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("rpent.utils.templates.get_output_dir", lambda: tmp_path) From ec6f04f35defbdc97994848eae62129956b9b848 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 16:34:52 +0800 Subject: [PATCH 15/18] test(interaction): keep prompt cleanup independently mergeable --- .../robots/dual_franka/test_dual_franka_extension.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py b/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py index 2c63f0257..2ea73b0b2 100644 --- a/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py +++ b/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py @@ -19,8 +19,10 @@ from pathlib import Path from robots.dual_franka import get_robot_spec +from robots.dual_franka.prompt_bundle import system_prompt, user_prompt from robots.dual_franka.runtime_config import load_runtime_config from robots.dual_franka.tasks import CLEAN_DESK_VLA_PROMPT, DUAL_FRANKA_TASKS +from rpent.prompt.utils import format_prompt from rpent.robots.base import enumerate_robots from rpent.robots.base import get_robot_spec as resolve_robot_spec @@ -123,9 +125,10 @@ def test_dual_franka_exploration_prompt_is_opt_in(dual_franka_run): dual_franka_run.args.task_id = 4 config = spec.parse_config(dual_franka_run.args) variables = {**config.prompt_vars, "output_dir": config.output_dir} - eval_prompt = spec.prompts.render("system", variables={**variables, "mode": "eval"}) - explore_prompt = spec.prompts.render("system", variables=variables) - explore_user_prompt = spec.prompts.render("user", variables=variables) + eval_vars = {**variables, "mode": "eval"} + eval_prompt = format_prompt(system_prompt(eval_vars), variables=eval_vars) + explore_prompt = format_prompt(system_prompt(variables), variables=variables) + explore_user_prompt = format_prompt(user_prompt(variables), variables=variables) assert "request_scene_reset" not in eval_prompt assert "request_scene_reset" in explore_prompt From bbd4d14d71c8cd76981223e297f8980cf5fd1b95 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 16:58:15 +0800 Subject: [PATCH 16/18] fix(interaction): preserve unaccepted input across session cancellation --- rpent/planner/claude_code.py | 31 ++++--- rpent/planner/codex.py | 9 ++- rpent/session/input.py | 33 +++++++- .../rpent/cli/test_main_contracts.py | 81 +++++++++++++++++++ 4 files changed, 134 insertions(+), 20 deletions(-) diff --git a/rpent/planner/claude_code.py b/rpent/planner/claude_code.py index 8b2f9f71c..0d9caba57 100644 --- a/rpent/planner/claude_code.py +++ b/rpent/planner/claude_code.py @@ -503,24 +503,31 @@ class _TerminalSessionAdapter: def __init__(self, *, toolkit: Toolkit, input_queue: InputQueue, emit_user) -> None: self._toolkit = toolkit - self._input_queue = SessionInputQueue(input_queue) + self._input_queue = SessionInputQueue(input_queue, retain_until_ack=True) self._emit_user = emit_user async def initial_query_succeeded(self, driver: _ClaudeSessionDriver) -> None: return None async def run(self, driver: _ClaudeSessionDriver) -> None: - while True: - nxt = await asyncio.to_thread(next_user_line, self._input_queue) - if self._input_queue.cancelled: - return - await cancel_and_wait(self._toolkit.cancel_active_and_wait) - await driver.interrupt() - if nxt is None: - return - self._toolkit.resume_calls() - self._emit_user(nxt) - await driver.query(nxt) + try: + while True: + nxt = await asyncio.to_thread(next_user_line, self._input_queue) + if self._input_queue.cancelled: + return + await cancel_and_wait(self._toolkit.cancel_active_and_wait) + await driver.interrupt() + if self._input_queue.cancelled: + return + if nxt is None: + self._input_queue.acknowledge() + return + self._toolkit.resume_calls() + await driver.query(nxt) + self._input_queue.acknowledge() + self._emit_user(nxt) + finally: + await self.close() async def on_message( self, diff --git a/rpent/planner/codex.py b/rpent/planner/codex.py index 9dc0e2601..d04cefe55 100644 --- a/rpent/planner/codex.py +++ b/rpent/planner/codex.py @@ -462,7 +462,7 @@ class _TerminalControl: """Forward terminal input through the same persistent Codex session driver.""" def __init__(self, input_queue: InputQueue, toolkit: Toolkit, emit_user) -> None: - self._input = SessionInputQueue(input_queue) + self._input = SessionInputQueue(input_queue, retain_until_ack=True) self._toolkit = toolkit self._emit_user = emit_user @@ -473,12 +473,13 @@ async def run(self, driver: _CodexSession) -> None: try: while True: text = await asyncio.to_thread(next_user_line, self._input) - if text is None: - return if self._input.cancelled: - self._input.put(text) + return + if text is None: + self._input.acknowledge() return await driver._submit_text(text) + self._input.acknowledge() self._emit_user(text) finally: self.end() diff --git a/rpent/session/input.py b/rpent/session/input.py index 35cf0526d..36ec760eb 100644 --- a/rpent/session/input.py +++ b/rpent/session/input.py @@ -25,13 +25,18 @@ class SessionInputQueue: A scope may wrap another scope, allowing a planner reply to end without cancelling its attended attempt or consuming the next attempt's input. + Terminal adapters retain reads until backend acceptance; cancelling that + consumer returns its pending input before later lines, even if a reader + thread has not yet returned to its awaiting coroutine. """ - def __init__(self, source: InputQueue) -> None: + def __init__(self, source: InputQueue, *, retain_until_ack: bool = False) -> None: """Wrap a source queue without taking ownership of its lifetime.""" self._source = source self._cancelled = threading.Event() self._lock = threading.Lock() + self._retain_until_ack = retain_until_ack + self._pending: list[str | None] = [] @property def cancelled(self) -> bool: @@ -39,9 +44,26 @@ def cancelled(self) -> bool: return self._cancelled.is_set() def cancel(self) -> None: - """Wake this scope's readers without closing or changing its source.""" + """Stop this consumer and return unaccepted input without inserting EOF.""" with self._lock: self._cancelled.set() + for item in reversed(self._pending): + self._return_unread(item) + self._pending.clear() + + def acknowledge(self) -> None: + """Release input after the backend successfully accepts it.""" + with self._lock: + self._pending.clear() + + def _return_unread(self, item: str | None) -> None: + """Return unaccepted input ahead of newer lines, through nested scopes.""" + if isinstance(self._source, SessionInputQueue): + self._source._return_unread(item) + else: + with self._source.mutex: + self._source.queue.appendleft(item) + self._source.not_empty.notify() def get(self, block: bool = True) -> str | None: """Read input, returning None when this scope or its source closes.""" @@ -50,14 +72,17 @@ def get(self, block: bool = True) -> str | None: if self.cancelled: return None try: - return self._source.get(block=False) + item = self._source.get(block=False) + if self._retain_until_ack: + self._pending.append(item) + return item except queue.Empty: if not block: raise self._cancelled.wait(0.1) def put(self, item: str | None) -> None: - """Forward real input to the source, including returned unread lines.""" + """Forward real input to the shared source.""" self._source.put(item) diff --git a/tests/unit_tests/rpent/cli/test_main_contracts.py b/tests/unit_tests/rpent/cli/test_main_contracts.py index 9f9be6d60..5a732564c 100644 --- a/tests/unit_tests/rpent/cli/test_main_contracts.py +++ b/tests/unit_tests/rpent/cli/test_main_contracts.py @@ -37,6 +37,87 @@ class ConfigCaptured(Exception): """Stop the CLI immediately after argument validation in routing tests.""" +@pytest.mark.parametrize("backend", ["claude_code", "codex"]) +@pytest.mark.parametrize("cancel_task", [False, True]) +def test_terminal_handoff_preserves_claimed_input_in_fifo( + monkeypatch, backend, cancel_task +): + import asyncio + import contextlib + import importlib + import queue + import threading + + from rpent.cli.tui import next_user_line + from rpent.session.input import SessionInputQueue + + module = importlib.import_module(f"rpent.planner.{backend}") + source = queue.Queue() + for line in (" ", " first steering ", "second steering", None): + source.put(line) + popped, release = threading.Event(), threading.Event() + deliveries = [] + + def held_read(scope): + value = next_user_line(scope) + popped.set() + assert release.wait(3) + return value + + async def submit(text): + deliveries.append(text) + + driver = SimpleNamespace( + interrupt=lambda: asyncio.sleep(0), query=submit, _submit_text=submit + ) + toolkit = SimpleNamespace( + cancel_active_and_wait=lambda: None, resume_calls=lambda: None + ) + + def adapter(): + if backend == "claude_code": + return module._TerminalSessionAdapter( + toolkit=toolkit, input_queue=source, emit_user=lambda text: None + ) + return module._TerminalControl(source, toolkit, lambda text: None) + + async def scenario(): + old = adapter() + monkeypatch.setattr(module, "next_user_line", held_read) + task = asyncio.create_task(old.run(driver)) + try: + assert await asyncio.to_thread(popped.wait, 2) + if cancel_task: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + elif backend == "claude_code": + await old.close() + else: + old.end() + # The next owner can read before the old thread returns, still in FIFO order. + current = SessionInputQueue(source) + assert next_user_line(current) == "first steering" + assert next_user_line(current) == "second steering" + assert next_user_line(current) is None + release.set() + with contextlib.suppress(asyncio.CancelledError): + await asyncio.wait_for(task, 2) + assert deliveries == [] and source.empty() + monkeypatch.setattr(module, "next_user_line", next_user_line) + source.put("accepted steering") + source.put(None) + await asyncio.wait_for(adapter().run(driver), 2) + assert deliveries == ["accepted steering"] and source.empty() + finally: + release.set() + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + + asyncio.run(scenario()) + + def _cli_module(): from rpent.cli import main as cli From 6294826d4802de00063c7e654c342790eea10d9f Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 17:07:04 +0800 Subject: [PATCH 17/18] fix(codex): resubmit input after a completed-turn steer rejection --- rpent/planner/codex.py | 20 +++++++- .../robots/dual_franka/test_exploration.py | 46 ++++++++++++++++--- .../rpent/planner/test_codex_contracts.py | 43 +++++++++++++++++ 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/rpent/planner/codex.py b/rpent/planner/codex.py index d04cefe55..054bf4ca1 100644 --- a/rpent/planner/codex.py +++ b/rpent/planner/codex.py @@ -539,8 +539,24 @@ async def _submit_text(self, text: str) -> int: if self._closing or self._thread is None: raise RuntimeError("Codex conversation is closed") if self._turn is not None: - await self._turn.steer(text) - return 0 + done = self._turn_done + assert done is not None + try: + await self._turn.steer(text) + return 0 + except openai_codex.JsonRpcError as exc: + if exc.code != -32600 or exc.message != "no active turn to steer": + raise + # The server can finish before its completion reaches the stream. + await done.wait() + if self.error is not None: + raise RuntimeError(self.error) from exc + if ( + self._closing + or self._recorder.finish_result is not None + or self._recorder.turns >= self._recorder.max_turns + ): + raise RuntimeError("Codex conversation is closed") from exc self._turn = await self._thread.turn(text, **self._turn_options) self._turn_done = asyncio.Event() self._turn_task = asyncio.create_task( diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index fcb04822b..d4a6711cc 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -698,9 +698,18 @@ def test_operator_evidence_remains_readable_after_success_memory_merge(setup, tm assert next_memory.authorize_read(evidence).read_bytes() == original -@pytest.mark.parametrize("ending", ["/success checked", None]) +@pytest.mark.parametrize( + ("completion", "ending"), + [ + ("normal", "/success checked"), + ("normal", None), + ("late", "/success checked"), + ("late", None), + ("failed", None), + ], +) def test_attended_confirmation_resumes_same_attempt_over_http( - setup, tmp_path, monkeypatch, ending + setup, tmp_path, monkeypatch, ending, completion ): import asyncio import queue @@ -748,18 +757,31 @@ class Client: def __init__(self, config): self.interrupted = asyncio.Event() + self.steering_rejected = asyncio.Event() clients.append(self) async def thread_start(self, **kwargs): return self async def turn(self, text, **kwargs): + if len(prompts) == 1: + assert ready.is_set() prompts.append(text) return self + async def steer(self, text): + from openai_codex import JsonRpcError + + assert completion != "normal" and text == "continue with this object" + self.steering_rejected.set() + raise JsonRpcError(-32600, "no active turn to steer") + async def stream(self): if len(prompts) == 1: await request("request_scene_reset", {"reason": "test"}) + if completion != "normal": + inputs.put("continue with this object") + await self.steering_rejected.wait() else: assert env.resets == 1 await request("move_delta", {"arm": "right", "delta_xyz": [0, 0, 0.01]}) @@ -770,7 +792,11 @@ async def stream(self): await self.interrupted.wait() yield { "method": "turn/completed", - "payload": {"turn": {"status": "completed"}}, + "payload": { + "turn": { + "status": "failed" if completion == "failed" else "completed" + } + }, } async def interrupt(self): @@ -812,11 +838,19 @@ def verdict(value, notes): assert prompted.wait(3) assert env.resets == 0 assert broker.route_line("/done") - assert ready.wait(3) - inputs.put("continue with this object") + if completion != "failed": + assert ready.wait(3) + if completion == "normal": + inputs.put("continue with this object") runner.join(5) - assert not runner.is_alive() and results[0].error is None + assert not runner.is_alive() assert len(clients) == 1 and clients[0].closed + if completion == "failed": + assert "Codex turn failed" in results[0].error + assert prompts == ["test\n\ntest task"] and not env.moves + assert inputs.get(block=False) == "continue with this object" + return + assert results[0].error is None assert prompts == ["test\n\ntest task", "continue with this object"] assert env.resets == 1 and len(env.moves) == 1 finally: diff --git a/tests/unit_tests/rpent/planner/test_codex_contracts.py b/tests/unit_tests/rpent/planner/test_codex_contracts.py index 6fbbebbd8..971a505af 100644 --- a/tests/unit_tests/rpent/planner/test_codex_contracts.py +++ b/tests/unit_tests/rpent/planner/test_codex_contracts.py @@ -863,6 +863,49 @@ def is_alive(self) -> bool: assert FakeMcpServer.instances[0].stopped is True +@pytest.mark.parametrize( + "error", + [ + JsonRpcError(-32600, "authentication required"), + JsonRpcError(-32600, "unsupported model"), + TransportClosedError("transport closed"), + ], +) +def test_terminal_rejected_input_preserves_error_and_fifo(error): + from rpent.planner.codex import _CodexSession, _Recorder, _TerminalControl + + async def run(): + source = queue.Queue() + for text in ("next object", "later feedback", None): + source.put(text) + control = _TerminalControl(source, FakeToolkit(), lambda text: None) + + async def reject(text): + raise error + + session = _CodexSession( + config=None, + thread_options={}, + turn_options={}, + recorder=_Recorder(max_turns=10, dashboard_events=RecordingSink()), + emit_event=lambda event: None, + control=control, + ) + session._thread = SimpleNamespace() # No new turn may start on this error. + session._turn = SimpleNamespace(steer=reject) + session._turn_done = asyncio.Event() + with pytest.raises(type(error)) as caught: + await asyncio.wait_for(control.run(session), 2) + assert caught.value is error + assert [source.get_nowait() for _ in range(3)] == [ + "next object", + "later feedback", + None, + ] + + asyncio.run(run()) + + def test_terminal_timeout_cancels_active_toolkit_work( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 68635bd57cf13ad8b08a14f5dece54cd0589fd63 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 17:34:07 +0800 Subject: [PATCH 18/18] test: narrow offline regressions to essential workflows --- .../unit_tests/robots/dual_franka/conftest.py | 80 +---- .../dual_franka/test_dashboard_operator.py | 84 +++-- .../dual_franka/test_dual_franka_extension.py | 35 +- .../dual_franka/test_dual_franka_tools.py | 68 +++- .../robots/dual_franka/test_exploration.py | 336 ++++++++++-------- .../rpent/cli/test_main_contracts.py | 27 +- .../rpent/planner/test_codex_contracts.py | 43 --- 7 files changed, 343 insertions(+), 330 deletions(-) diff --git a/tests/unit_tests/robots/dual_franka/conftest.py b/tests/unit_tests/robots/dual_franka/conftest.py index d00a2fc1c..8728fc308 100644 --- a/tests/unit_tests/robots/dual_franka/conftest.py +++ b/tests/unit_tests/robots/dual_franka/conftest.py @@ -15,11 +15,8 @@ """Shared fixtures for offline dual-Franka tests.""" from collections.abc import Iterator -from dataclasses import replace from pathlib import Path -from types import SimpleNamespace -import numpy as np import pytest import yaml @@ -27,84 +24,11 @@ from robots.franka.runtime_config import set_robot_config_path -class FakeEnv: - def __init__(self): - self.meta = {} - self.resets = 0 - self.moves = [] - self.reset_result = {"ok": True} - self.fail_observation = False - - def reset(self): - self.resets += 1 - return self.reset_result - - def get_observation(self): - if self.fail_observation: - raise RuntimeError("camera offline") - return { - "main_images": np.full((8, 8, 3), self.resets, dtype=np.uint8), - "extra_view_images": np.zeros((2, 8, 8, 3), dtype=np.uint8), - "main_depths": np.ones((8, 8), dtype=np.float32), - "extra_view_depths": np.ones((2, 8, 8), dtype=np.float32), - "d455_images": np.zeros((8, 8, 3), dtype=np.uint8), - "d455_depths": np.ones((8, 8), dtype=np.float32), - } - - def get_robot_state(self): - return { - "left_arm": {"tcp_pose": [0.5, 0, 0.5, 0, 0, 0, 1]}, - "right_arm": {"tcp_pose": [0.5, 0, 0.5, 0, 0, 0, 1]}, - } - - def get_camera_meta(self): - return {"cameras": {}} - - def move_delta(self, arm, delta): - self.moves.append((arm, list(delta))) - return {"ok": True} - - @pytest.fixture def fake_env(): - return FakeEnv() - - -@pytest.fixture -def dual_franka_run(tmp_path, dual_franka_robot_config, fake_env): - from robots.dual_franka import robot_spec - from rpent.cli import main + from tests.unit_tests.robots.dual_franka.test_exploration import FakeEnv - calls = [] - - def init_runtime(*args): - calls.append(args) - return [], {"env": fake_env, "model": None, "task_description": "test"} - - run = SimpleNamespace( - env=fake_env, - spec=replace(robot_spec.get_robot_spec(), init_runtime=init_runtime), - runtime_calls=calls, - argv=[ - "rpent", - "--robot", - "dual_franka", - "--planner", - "codex", - "--explore", - "--auto-merge-memory", - "--robot-config", - str(dual_franka_robot_config), - "--memory-dir", - str(tmp_path / "memory"), - "--output-dir", - str(tmp_path / "run"), - ], - ) - parser = main._build_argparser() - run.spec.add_cli_args(parser, use_dashboard=False) - run.args = parser.parse_args(run.argv[1:]) - return run + return FakeEnv() @pytest.fixture diff --git a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py index 1fec241a3..99897251d 100644 --- a/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py +++ b/tests/unit_tests/robots/dual_franka/test_dashboard_operator.py @@ -16,10 +16,12 @@ import time from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace from types import SimpleNamespace from fastapi.testclient import TestClient +from robots.dual_franka.robot_spec import get_robot_spec from rpent.cli import dashboard from rpent.dashboard.server import DashboardServer from rpent.dashboard.state import DashboardState @@ -27,13 +29,35 @@ def test_web_operator_confirms_reset_and_completes_task( - tmp_path, monkeypatch, dual_franka_run + tmp_path, monkeypatch, dual_franka_robot_config, fake_env ): - spec = dual_franka_run.spec - args = dual_franka_run.args - args.dashboard = True - args.task_id = 3 - env = dual_franka_run.env + env = fake_env + spec = replace( + get_robot_spec(), + init_runtime=lambda *args: ( + [], + {"env": env, "model": None, "task_description": "test"}, + ), + ) + from rpent.cli.main import _build_argparser + + parser = _build_argparser() + spec.add_cli_args(parser, use_dashboard=True) + args = parser.parse_args( + [ + "--robot", + "dual_franka", + "--dashboard", + "--explore", + "--task-id", + "3", + "--robot-config", + str(dual_franka_robot_config), + "--memory-dir", + str(tmp_path / "memory"), + "--auto-merge-memory", + ] + ) state = DashboardState(output_dir=tmp_path, dashboard_spec=spec.dashboard) state.shared_services_ready() state.request_task({"task_id": 3}) @@ -58,49 +82,39 @@ def solve(**kwargs): "generation": operator["generation"], "request_id": operator["pending"]["id"], } + + def send(text, **context): + return client.post( + "/api/session/messages", + json={"text": text, "operator_context": context}, + ) + for stale in ( {**context, "generation": context["generation"] - 1}, {**context, "request_id": "expired"}, ): - assert client.post( - "/api/session/messages", - json={"text": "/done", "operator_context": stale}, - ).is_error + assert send("/done", **stale).is_error assert env.resets == 0 - assert ( - client.post( - "/api/session/messages", - json={"text": "/done", "operator_context": context}, - ).status_code - == 202 - ) + assert send("/done", **context).status_code == 202 assert not future.result(3).is_error toolkit.execute_tool( "move_delta", {"arm": "right", "delta_xyz": [0, 0, 0.01]} ) - assert ( - client.post( - "/api/session/messages", - json={"text": "/success", "operator_context": context}, - ).status_code - == 202 - ) + assert send("/success", **context).status_code == 202 return PlannerResult() monkeypatch.setattr( dashboard, "build_planner", lambda *a, **kw: SimpleNamespace(solve=solve) ) - assert ( - dashboard._run_dashboard_task( - args=args, - robot_spec=spec, - state=state, - claimed=claimed, - shared_runtime_kwargs={}, - unique_components={"env"}, - session_root=tmp_path, - ) - is None + result = dashboard._run_dashboard_task( + args=args, + robot_spec=spec, + state=state, + claimed=claimed, + shared_runtime_kwargs={}, + unique_components={"env"}, + session_root=tmp_path, ) + assert result is None assert env.resets == 1 and len(env.moves) == 1 assert (tmp_path / "memory/task-specific/dual_franka_t3_recipe.jsonl").is_file() diff --git a/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py b/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py index 2ea73b0b2..72ffa752b 100644 --- a/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py +++ b/tests/unit_tests/robots/dual_franka/test_dual_franka_extension.py @@ -120,17 +120,34 @@ def test_metal_basket_task_registers_non_sorting_prompt(): assert "green object before the blue object" in constraints -def test_dual_franka_exploration_prompt_is_opt_in(dual_franka_run): - spec = dual_franka_run.spec - dual_franka_run.args.task_id = 4 - config = spec.parse_config(dual_franka_run.args) - variables = {**config.prompt_vars, "output_dir": config.output_dir} - eval_vars = {**variables, "mode": "eval"} +def test_dual_franka_exploration_prompt_is_opt_in(): + eval_vars = {"mode": "eval"} + explore_vars = { + "mode": "explore", + "task_id": 4, + "session_number": 1, + "session_max": 1, + "recipe_tag": "dual_franka_t4", + "memory_dir": "/tmp/memory", + "task_name": "offline", + "instruction": "test instruction", + "setup": "test setup", + "success_criteria": "test success", + "constraints": "1. test constraint", + "output_dir": "/tmp/run", + "memory_inbox": "/tmp/memory/_internal/inbox/dual_franka_t4", + } eval_prompt = format_prompt(system_prompt(eval_vars), variables=eval_vars) - explore_prompt = format_prompt(system_prompt(variables), variables=variables) - explore_user_prompt = format_prompt(user_prompt(variables), variables=variables) + explore_prompt = format_prompt( + system_prompt(explore_vars), + variables=explore_vars, + ) + explore_user_prompt = format_prompt( + user_prompt(explore_vars), + variables=explore_vars, + ) assert "request_scene_reset" not in eval_prompt assert "request_scene_reset" in explore_prompt assert "request_operator_verdict" in explore_prompt - assert variables["memory_inbox"] in explore_user_prompt + assert "/tmp/memory/_internal/inbox/dual_franka_t4" in explore_user_prompt diff --git a/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py b/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py index cc97ebb1a..1e111834f 100644 --- a/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py +++ b/tests/unit_tests/robots/dual_franka/test_dual_franka_tools.py @@ -25,7 +25,7 @@ import numpy as np import pytest -from robots.dual_franka import get_robot_spec +from robots.dual_franka import get_robot_spec, perception from robots.dual_franka.perception import ( back_project, load_calibration_bundle, @@ -643,7 +643,8 @@ def fail_json_save(name, value, **kwargs): assert "_image_" not in text_block["text"] -def test_segment_returns_mask_overlay_and_world_point(tmp_path: Path): +@pytest.mark.parametrize("parallel", [False, True]) +def test_segment_returns_mask_overlay_and_world_point(tmp_path, monkeypatch, parallel): state = EnvState(tmp_path) with state.record_step( state={ @@ -690,25 +691,72 @@ def test_segment_returns_mask_overlay_and_world_point(tmp_path: Path): " - [0.0, 0.0, 1.0, 0.0]\n" " - [0.0, 0.0, 0.0, 1.0]\n" ) + targets = ["first", "second"] if parallel else ["first"] + barrier = threading.Barrier(len(targets)) + localize = perception._mask_to_camera_world + + def synchronized_localize(*args, **kwargs): + # Both calls reach localization before either can save an artifact. + barrier.wait(timeout=3) + return localize(*args, **kwargs) + + monkeypatch.setattr(perception, "_mask_to_camera_world", synchronized_localize) + save = state.save + save_barrier = threading.Barrier(len(targets)) + + def synchronized_save(name, value, **kwargs): + if name.startswith("d455_segment_overlay_"): + save_barrier.wait(timeout=3) + return save(name, value, **kwargs) + + monkeypatch.setattr(state, "save", synchronized_save) set_robot_config_path(config) try: - result_native = segment( + call = partial( + segment, prompt="white cardboard box interior", - target_name="cardboard_box_interior", min_valid_depth_pixels=1, state=state, sam3_client=FakeSam3Client(), ) + with ThreadPoolExecutor(max_workers=len(targets)) as pool: + results = list(pool.map(lambda target: call(target_name=target), targets)) + result_native = results[0] result = result_native.data + + monkeypatch.setattr(perception, "_mask_to_camera_world", localize) + failed_names = [] + + def fail_json_save(name, value, **kwargs): + if name.startswith("d455_segment_") and name.endswith(".json"): + failed_names.append(name) + return None + return save(name, value, **kwargs) + + monkeypatch.setattr(state, "save", fail_json_save) + failed = call(target_name="failed") + failed_overlay = state.load_bytes(failed.data["overlay_artifact"]) + monkeypatch.setattr(state, "save", save) + following = call(target_name="following") + assert failed.is_error + assert not following.is_error + assert following.data["segment_artifact"] not in failed_names + assert following.data["overlay_artifact"] != failed.data["overlay_artifact"] + assert state.load_bytes(failed.data["overlay_artifact"]) == failed_overlay + assert ( + state.load(following.data["segment_artifact"])["target_name"] == "following" + ) state.save("d455_depth.npy", np.full((8, 8), 3.958), step=step) - rejected = segment( - prompt="white cardboard box interior", - state=state, - sam3_client=FakeSam3Client(), - ).data + assert call(target_name="outside_workspace").data["point_xyz"] is None finally: set_robot_config_path(None) + assert len({item.data["segment_artifact"] for item in results}) == len(targets) + assert len({item.data["overlay_artifact"] for item in results}) == len(targets) + for target, item in zip(targets, results, strict=True): + assert not item.is_error + assert state.load(item.data["segment_artifact"])["target_name"] == target + assert item.images == [state.load_bytes(item.data["overlay_artifact"])] assert result["ok"] assert result["found"] assert result["coordinate_frame"] == "right_base" @@ -720,8 +768,6 @@ def test_segment_returns_mask_overlay_and_world_point(tmp_path: Path): assert result["segment_artifact"].startswith("d455_segment_") assert result["overlay_artifact"].startswith("d455_segment_overlay_") assert result_native.images and all(result_native.images) - - assert rejected["point_xyz"] is None assert result["image_block_order"] == ["d455_segment_overlay"] blocks = mcp_result(result_native)["content"] diff --git a/tests/unit_tests/robots/dual_franka/test_exploration.py b/tests/unit_tests/robots/dual_franka/test_exploration.py index d4a6711cc..c1e8acb71 100644 --- a/tests/unit_tests/robots/dual_franka/test_exploration.py +++ b/tests/unit_tests/robots/dual_franka/test_exploration.py @@ -14,6 +14,7 @@ """Offline attended exploration: real toolkit, logger and memory, fake hardware.""" +import argparse import json from pathlib import Path @@ -26,6 +27,44 @@ from rpent.memory import MemoryManager +class FakeEnv: + def __init__(self): + self.meta = {} + self.resets = 0 + self.moves = [] + self.reset_result = {"ok": True} + self.fail_observation = False + + def reset(self): + self.resets += 1 + return self.reset_result + + def get_observation(self): + if self.fail_observation: + raise RuntimeError("camera offline") + return { + "main_images": np.full((8, 8, 3), self.resets, dtype=np.uint8), + "extra_view_images": np.zeros((2, 8, 8, 3), dtype=np.uint8), + "main_depths": np.ones((8, 8), dtype=np.float32), + "extra_view_depths": np.ones((2, 8, 8), dtype=np.float32), + "d455_images": np.zeros((8, 8, 3), dtype=np.uint8), + "d455_depths": np.ones((8, 8), dtype=np.float32), + } + + def get_robot_state(self): + return { + "left_arm": {"tcp_pose": [0.5, 0, 0.5, 0, 0, 0, 1]}, + "right_arm": {"tcp_pose": [0.5, 0, 0.5, 0, 0, 0, 1]}, + } + + def get_camera_meta(self): + return {"cameras": {}} + + def move_delta(self, arm, delta): + self.moves.append((arm, list(delta))) + return {"ok": True} + + @pytest.fixture def setup(tmp_path, monkeypatch, dual_franka_robot_config, fake_env): from rpent.utils import logging @@ -151,9 +190,28 @@ def test_budget_and_old_perception_boundary(setup): assert env.resets == 2 -def test_explore_prompt_and_factory_use_local_layered_memory(tmp_path, dual_franka_run): - config = dual_franka_run.spec.parse_config(dual_franka_run.args) - prompt = dual_franka_run.spec.prompts.render( +def test_explore_prompt_and_factory_use_local_layered_memory( + tmp_path, dual_franka_robot_config, fake_env +): + parser = argparse.ArgumentParser() + parser.add_argument("--explore", action="store_true") + parser.add_argument("--memory-dir") + parser.add_argument("--memory-profile", default=None) + parser.add_argument("--output-dir") + robot_spec.get_robot_spec().add_cli_args(parser, False) + args = parser.parse_args( + [ + "--explore", + "--memory-dir", + str(tmp_path / "memory"), + "--output-dir", + str(tmp_path), + "--robot-config", + str(dual_franka_robot_config), + ] + ) + config = robot_spec.get_robot_spec().parse_config(args) + prompt = robot_spec.get_robot_spec().prompts.render( "system", variables={**config.prompt_vars, "output_dir": tmp_path} ) assert "request_scene_reset" in prompt and "request_operator_verdict" in prompt @@ -164,11 +222,7 @@ def test_explore_prompt_and_factory_use_local_layered_memory(tmp_path, dual_fran ) assert "{{" not in prompt and "libero_terminated" not in prompt t = robot_spec.get_toolkit( - runtime_kwargs={ - "env": dual_franka_run.env, - "model": None, - "task_description": "test", - }, + runtime_kwargs={"env": fake_env, "model": None, "task_description": "test"}, dashboard_events=NullDashboardEventSink(), config=config, mode="exploration", @@ -189,10 +243,23 @@ def test_explore_prompt_and_factory_use_local_layered_memory(tmp_path, dual_fran def test_successful_memory_pair_uses_existing_merge_and_index(setup, tmp_path): + from rpent.cli.attended import _finalize_operator_verdict + from rpent.planner.base import PlannerResult + t, _, replies = setup reset(t, replies) call(t, "move_delta", arm="left", delta_xyz=[0.01, 0, 0]) - verdict(t, replies, "success") + assert t.request_direct_verdict("success") + state_dir = tmp_path / "sessions" / "session_001" + finish, error, handoff = _finalize_operator_verdict( + PlannerResult(), + finalize_verdict=t.finalize_direct_verdict, + state_output_dir=state_dir, + session_number=1, + ) + evidence = state_dir / "operator_outcome.json" + assert finish["status"] == "success" and error is None and str(evidence) in handoff + original = evidence.read_bytes() t.write_recipe("dual_franka_t0") inbox = t.memory.root / "_internal/inbox/dual_franka_t0" inbox.mkdir(parents=True) @@ -228,19 +295,25 @@ def test_successful_memory_pair_uses_existing_merge_and_index(setup, tmp_path): assert not t.memory.validate() assert (t.memory.root / "MEMORY.md").exists() assert (t.memory.root / "task-specific/dual_franka_t0_recipe.jsonl").exists() + next_memory = MemoryManager( + t.memory.root, memory_access="inbox_write", inbox_cell_tag="dual_franka_t0" + ) + assert next_memory.authorize_read(evidence).read_bytes() == original @pytest.mark.parametrize("interactive", [False, True]) def test_cli_two_sessions_operator_feedback_and_memory_pipeline( - tmp_path, monkeypatch, interactive, dual_franka_run + tmp_path, monkeypatch, interactive, dual_franka_robot_config, fake_env ): import sys + from dataclasses import replace from types import SimpleNamespace from rpent.cli import main as cli from rpent.tools.human_in_the_loop import HumanInTheLoopInput - env = dual_franka_run.env + env = fake_env + runtimes = [] planners = [] replies = iter(["done", "failure dropped", "done", "success lifted"]) @@ -294,7 +367,16 @@ def solve(self, *, toolkit, system_prompt, user_message, **kwargs): finish_result=finish, messages=[], stats={}, error=None ) - monkeypatch.setattr(cli, "get_robot_spec", lambda name: dual_franka_run.spec) + def init_runtime(*args): + runtimes.append(args) + return [], { + "env": env, + "model": None, + "task_description": "test", + } + + spec = replace(robot_spec.get_robot_spec(), init_runtime=init_runtime) + monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) monkeypatch.setattr(cli, "build_planner", Planner) monkeypatch.setattr( cli, @@ -307,17 +389,26 @@ def solve(self, *, toolkit, system_prompt, user_message, **kwargs): sys, "argv", [ - *dual_franka_run.argv, + "rpent", + "--robot", + "dual_franka", + "--explore", "--explore-sessions", "2", "--explore-attempts-per-session", "1", + "--output-dir", + str(tmp_path / "run"), + "--memory-dir", + str(tmp_path / "memory"), + "--robot-config", + str(dual_franka_robot_config), + "--auto-merge-memory", *(["--interactive"] if interactive else []), ], ) assert cli.main() == 0 - assert len(planners) == 2 and len(dual_franka_run.runtime_calls) == 1 - assert env.resets == 2 + assert len(planners) == 2 and len(runtimes) == 1 and env.resets == 2 traces = [ json.loads((tmp_path / f"run/sessions/session_{i:03d}/states.json").read_text()) for i in (1, 2) @@ -495,16 +586,19 @@ def test_direct_success_with_failed_observation_does_not_publish(setup): def test_cli_direct_verdict_finalizes_and_merges_only_without_errors( tmp_path, monkeypatch, - dual_franka_run, + dual_franka_robot_config, + fake_env, verdict, planner_error, robot_name, ): import sys + from dataclasses import replace from types import SimpleNamespace from rpent.cli import main as cli + env = fake_env handlers = {} command = verdict verdict = "abort" if command == "eof" else verdict @@ -532,7 +626,18 @@ def solve(self, *, toolkit, input_queue, **kwargs): finish_result=None, messages=[], stats={}, error=planner_error ) - monkeypatch.setattr(cli, "get_robot_spec", lambda name: dual_franka_run.spec) + spec = replace( + robot_spec.get_robot_spec(), + init_runtime=lambda *args: ( + [], + { + "env": env, + "model": None, + "task_description": "test", + }, + ), + ) + monkeypatch.setattr(cli, "get_robot_spec", lambda name: spec) original_toolkit_factory = cli.get_toolkit monkeypatch.setattr( cli, @@ -541,8 +646,26 @@ def solve(self, *, toolkit, input_queue, **kwargs): ) monkeypatch.setattr(cli, "build_planner", lambda *args, **kwargs: Planner()) monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True)) - dual_franka_run.argv[dual_franka_run.argv.index("--robot") + 1] = robot_name - monkeypatch.setattr(sys, "argv", [*dual_franka_run.argv, "--interactive"]) + monkeypatch.setattr( + sys, + "argv", + [ + "rpent", + "--robot", + robot_name, + "--planner", + "codex", + "--explore", + "--auto-merge-memory", + "--interactive", + "--output-dir", + str(tmp_path / "run"), + "--memory-dir", + str(tmp_path / "memory"), + "--robot-config", + str(dual_franka_robot_config), + ], + ) assert cli.main() == (1 if planner_error and verdict != "failure" else 0) assert (tmp_path / "memory/task-specific/dual_franka_t0.json").is_file() == ( verdict == "success" and planner_error is None @@ -591,33 +714,6 @@ def test_explore_task_is_only_in_user_prompt(): assert section in user -def test_current_view_refreshes_but_historical_view_does_not(setup): - t, env, replies = setup - reset(t, replies) - old_step = t.state.latest_step - reads = [] - original = env.get_observation - - def observe(): - reads.append(True) - obs = original() - obs["d455_images"][:] = 77 - return obs - - env.get_observation = observe - observed = t.execute_tool("view_env_state", {}) - assert len(reads) == 1 and t.state.latest_step == old_step + 1 - assert observed.images == [t.state.load_bytes("d455.png")] - assert np.all(t.state.load("d455.png") == 77) - call(t, "view_env_state", step=old_step) - assert len(reads) == 1 - assert env.resets == 1 and not env.moves - call(t, "move_delta", arm="left", delta_xyz=[0.01, 0, 0]) - assert len(reads) == 2 and t.state.latest_step == old_step + 2 - assert len(env.moves) == 1 and env.moves[0][0] == "left" - np.testing.assert_allclose(env.moves[0][1], [0.01, 0, 0]) - - @pytest.mark.parametrize("second_tool", ["view_env_state", "request_operator_verdict"]) def test_fresh_observation_and_verdict_serialize_state_writes( setup, monkeypatch, second_tool @@ -650,54 +746,15 @@ def hold_first_image(name, value, **kwargs): release.set() assert not first.result(5).is_error assert not second.result(5).is_error + assert first.result().images == [ + t.state.load_bytes("d455.png", step=initial_step + 1) + ] + call(t, "view_env_state", step=initial_step) assert t.state.latest_step == initial_step + 2 and t._scene_ready assert not call(t, "move_delta", arm="left", delta_xyz=[0.01, 0, 0]).get("error") assert len(env.moves) == 1 and env.resets == 1 -def test_operator_evidence_remains_readable_after_success_memory_merge(setup, tmp_path): - from rpent.cli.attended import _finalize_operator_verdict, _publish_success_recipe - from rpent.planner.base import PlannerResult - - t, _, replies = setup - reset(t, replies) - call(t, "move_delta", arm="left", delta_xyz=[0.01, 0, 0]) - tag = "dual_franka_t0" - note = t.memory.authorize_write( - t.memory.root / "_internal" / "inbox" / tag / "strategy.md" - ) - note.parent.mkdir(parents=True, exist_ok=True) - note.write_text( - "---\nscope: global\nkind: strategy\ntitle: example\napplies_when: example\n" - "confidence: single-shot\nevidence:\n cells: [dual_franka_t0]\n---\nObserved example.\n" - ) - assert t.request_direct_verdict("success") - state_dir = tmp_path / "sessions" / "session_001" - finish, error, handoff = _finalize_operator_verdict( - PlannerResult(), - finalize_verdict=t.finalize_direct_verdict, - state_output_dir=state_dir, - session_number=1, - ) - evidence = state_dir / "operator_outcome.json" - assert finish["status"] == "success" and error is None - assert str(evidence) in handoff - original = evidence.read_bytes() - t.write_recipe(tag) - _publish_success_recipe( - t.memory, - recipe_tag=tag, - output_dir=tmp_path, - state_output_dir=state_dir, - auto_merge=True, - ) - assert (t.memory.root / "global" / "strategy.md").is_file() - next_memory = MemoryManager( - t.memory.root, memory_access="inbox_write", inbox_cell_tag=tag - ) - assert next_memory.authorize_read(evidence).read_bytes() == original - - @pytest.mark.parametrize( ("completion", "ending"), [ @@ -706,24 +763,25 @@ def test_operator_evidence_remains_readable_after_success_memory_merge(setup, tm ("late", "/success checked"), ("late", None), ("failed", None), + ("authentication required", None), + ("unsupported model", None), + ("transport closed", None), ], ) -def test_attended_confirmation_resumes_same_attempt_over_http( +def test_attended_confirmation_resumes_same_attempt( setup, tmp_path, monkeypatch, ending, completion ): import asyncio import queue import threading - import httpx - from mcp import ClientSession - from mcp.client.streamable_http import streamable_http_client + from openai_codex import JsonRpcError, TransportClosedError from rpent.planner.codex import CodexPlanner from rpent.session.input import SessionInputQueue from rpent.tools.human_in_the_loop import HumanInTheLoopInput - toolkit, env, replies = setup + toolkit, env, _ = setup broker = HumanInTheLoopInput(interactive=True) toolkit._operator_input = broker ready, prompted = threading.Event(), threading.Event() @@ -736,21 +794,10 @@ def test_attended_confirmation_resumes_same_attempt_over_http( ) inputs = SessionInputQueue(queue.Queue()) clients, prompts = [], [] - endpoint = [] async def request(name, args): - async with ( - httpx.AsyncClient(trust_env=False) as client, - streamable_http_client(endpoint[0], http_client=client) as ( - read, - write, - _, - ), - ClientSession(read, write) as session, - ): - await session.initialize() - result = await session.call_tool(name, args) - assert not result.isError + result = await asyncio.to_thread(toolkit.execute_tool, name, args) + assert not result.is_error class Client: closed = False @@ -764,17 +811,20 @@ async def thread_start(self, **kwargs): return self async def turn(self, text, **kwargs): - if len(prompts) == 1: - assert ready.is_set() prompts.append(text) return self async def steer(self, text): - from openai_codex import JsonRpcError - assert completion != "normal" and text == "continue with this object" self.steering_rejected.set() - raise JsonRpcError(-32600, "no active turn to steer") + if completion == "transport closed": + raise TransportClosedError(completion) + raise JsonRpcError( + -32600, + "no active turn to steer" + if completion in {"late", "failed"} + else completion, + ) async def stream(self): if len(prompts) == 1: @@ -819,41 +869,43 @@ def verdict(value, notes): timeout_s=10, dashboard_events=NullDashboardEventSink(), ) - monkeypatch.setattr(planner, "_build_config", lambda url: endpoint.append(url)) + monkeypatch.setattr(planner, "_build_config", lambda url: None) - results = [] - runner = threading.Thread( - target=lambda: results.append( - planner.solve( + async def run(): + runner = asyncio.create_task( + planner._solve_interactive( toolkit=toolkit, input_queue=inputs, - system_prompt="test", - user_message="test task", + prompt="test\n\ntest task", + initial_user_text="test task", max_turns=10, + interaction=None, ) ) - ) - runner.start() - try: - assert prompted.wait(3) - assert env.resets == 0 - assert broker.route_line("/done") - if completion != "failed": - assert ready.wait(3) - if completion == "normal": - inputs.put("continue with this object") - runner.join(5) - assert not runner.is_alive() - assert len(clients) == 1 and clients[0].closed - if completion == "failed": - assert "Codex turn failed" in results[0].error - assert prompts == ["test\n\ntest task"] and not env.moves - assert inputs.get(block=False) == "continue with this object" - return - assert results[0].error is None + try: + assert await asyncio.to_thread(prompted.wait, 3) + assert env.resets == 0 and broker.route_line("/done") + if completion in {"normal", "late"}: + assert await asyncio.to_thread(ready.wait, 3) + if completion == "normal": + inputs.put("continue with this object") + return await asyncio.wait_for(runner, 5) + finally: + inputs.put(None) + broker.close() + if not runner.done(): + runner.cancel() + await asyncio.gather(runner, return_exceptions=True) + + result = asyncio.run(run()) + assert len(clients) == 1 and clients[0].closed + if completion not in {"normal", "late"}: + assert ( + "Codex turn failed" if completion == "failed" else completion + ) in result.error + assert prompts == ["test\n\ntest task"] and not env.moves + assert inputs.get(block=False) == "continue with this object" + else: + assert result.error is None assert prompts == ["test\n\ntest task", "continue with this object"] assert env.resets == 1 and len(env.moves) == 1 - finally: - inputs.put(None) - broker.close() - runner.join(5) diff --git a/tests/unit_tests/rpent/cli/test_main_contracts.py b/tests/unit_tests/rpent/cli/test_main_contracts.py index 5a732564c..36ce95874 100644 --- a/tests/unit_tests/rpent/cli/test_main_contracts.py +++ b/tests/unit_tests/rpent/cli/test_main_contracts.py @@ -43,7 +43,6 @@ def test_terminal_handoff_preserves_claimed_input_in_fifo( monkeypatch, backend, cancel_task ): import asyncio - import contextlib import importlib import queue import threading @@ -89,31 +88,35 @@ async def scenario(): assert await asyncio.to_thread(popped.wait, 2) if cancel_task: task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task + await asyncio.gather(task, return_exceptions=True) elif backend == "claude_code": await old.close() else: old.end() # The next owner can read before the old thread returns, still in FIFO order. current = SessionInputQueue(source) - assert next_user_line(current) == "first steering" - assert next_user_line(current) == "second steering" - assert next_user_line(current) is None + assert [next_user_line(current) for _ in range(3)] == [ + "first steering", + "second steering", + None, + ] release.set() - with contextlib.suppress(asyncio.CancelledError): - await asyncio.wait_for(task, 2) + result = await asyncio.wait_for( + asyncio.gather(task, return_exceptions=True), 2 + ) + assert result == [None] or ( + cancel_task and isinstance(result[0], asyncio.CancelledError) + ) assert deliveries == [] and source.empty() monkeypatch.setattr(module, "next_user_line", next_user_line) - source.put("accepted steering") - source.put(None) + for line in ("accepted steering", None): + source.put(line) await asyncio.wait_for(adapter().run(driver), 2) assert deliveries == ["accepted steering"] and source.empty() finally: release.set() task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task + await asyncio.gather(task, return_exceptions=True) asyncio.run(scenario()) diff --git a/tests/unit_tests/rpent/planner/test_codex_contracts.py b/tests/unit_tests/rpent/planner/test_codex_contracts.py index 971a505af..6fbbebbd8 100644 --- a/tests/unit_tests/rpent/planner/test_codex_contracts.py +++ b/tests/unit_tests/rpent/planner/test_codex_contracts.py @@ -863,49 +863,6 @@ def is_alive(self) -> bool: assert FakeMcpServer.instances[0].stopped is True -@pytest.mark.parametrize( - "error", - [ - JsonRpcError(-32600, "authentication required"), - JsonRpcError(-32600, "unsupported model"), - TransportClosedError("transport closed"), - ], -) -def test_terminal_rejected_input_preserves_error_and_fifo(error): - from rpent.planner.codex import _CodexSession, _Recorder, _TerminalControl - - async def run(): - source = queue.Queue() - for text in ("next object", "later feedback", None): - source.put(text) - control = _TerminalControl(source, FakeToolkit(), lambda text: None) - - async def reject(text): - raise error - - session = _CodexSession( - config=None, - thread_options={}, - turn_options={}, - recorder=_Recorder(max_turns=10, dashboard_events=RecordingSink()), - emit_event=lambda event: None, - control=control, - ) - session._thread = SimpleNamespace() # No new turn may start on this error. - session._turn = SimpleNamespace(steer=reject) - session._turn_done = asyncio.Event() - with pytest.raises(type(error)) as caught: - await asyncio.wait_for(control.run(session), 2) - assert caught.value is error - assert [source.get_nowait() for _ in range(3)] == [ - "next object", - "later feedback", - None, - ] - - asyncio.run(run()) - - def test_terminal_timeout_cancels_active_toolkit_work( tmp_path: Path, monkeypatch: pytest.MonkeyPatch,