DRAFT - Do Not Review - UR10e Deploy Gear Insertion Environment

scripts/reinforcement_learning/rsl_rl/train.py

@@ -56,7 +56,7 @@
 from packaging import version
 
 # check minimum supported rsl-rl version
-RSL_RL_VERSION = "3.0.1"
+RSL_RL_VERSION = "3.1.0"
 installed_version = metadata.version("rsl-rl-lib")
 if version.parse(installed_version) < version.parse(RSL_RL_VERSION):
     if platform.system() == "Windows":

source/isaaclab/isaaclab/envs/mdp/rewards.py

@@ -241,11 +241,17 @@ def applied_torque_limits(env: ManagerBasedRLEnv, asset_cfg: SceneEntityCfg = Sc
     )
     return torch.sum(out_of_limits, dim=1)
 
+def action_rate(env: ManagerBasedRLEnv) -> torch.Tensor:
+    """Penalize the rate of change of the actions using L2 norm."""
+    return torch.norm(env.action_manager.action - env.action_manager.prev_action, p=2, dim=-1)
 
 def action_rate_l2(env: ManagerBasedRLEnv) -> torch.Tensor:
     """Penalize the rate of change of the actions using L2 squared kernel."""
     return torch.sum(torch.square(env.action_manager.action - env.action_manager.prev_action), dim=1)
 
+def action(env: ManagerBasedRLEnv) -> torch.Tensor:
+    """Penalize the actions using L2 squared kernel (summed for each environment)."""
+    return torch.sum(env.action_manager.action**2, dim=-1)
 
 def action_l2(env: ManagerBasedRLEnv) -> torch.Tensor:
     """Penalize the actions using L2 squared kernel."""

source/isaaclab/isaaclab/utils/noise/__init__.py

@@ -26,8 +26,8 @@
 
 """
 from .noise_cfg import NoiseCfg  # noqa: F401
-from .noise_cfg import ConstantNoiseCfg, GaussianNoiseCfg, NoiseModelCfg, NoiseModelWithAdditiveBiasCfg, UniformNoiseCfg
-from .noise_model import NoiseModel, NoiseModelWithAdditiveBias, constant_noise, gaussian_noise, uniform_noise
+from .noise_cfg import ConstantNoiseCfg, GaussianNoiseCfg, NoiseModelCfg, NoiseModelWithAdditiveBiasCfg, UniformNoiseCfg, ResetSampledNoiseModelCfg
+from .noise_model import NoiseModel, NoiseModelWithAdditiveBias, ResetSampledNoiseModel, constant_noise, gaussian_noise, uniform_noise
 
 # Backward compatibility
 ConstantBiasNoiseCfg = ConstantNoiseCfg

source/isaaclab/isaaclab/utils/noise/noise_cfg.py

@@ -109,3 +109,15 @@ class NoiseModelWithAdditiveBiasCfg(NoiseModelCfg):
 
     Defaults to True.
     """
+
+@configclass
+class ResetSampledNoiseModelCfg(NoiseModelCfg):
+    """Configuration for a noise model that samples noise ONLY during reset."""
+
+    class_type: type = noise_model.ResetSampledNoiseModel
+
+    noise_cfg: NoiseCfg = MISSING
+    """The noise configuration for the noise.
+
+    Based on this configuration, the noise is sampled at every reset of the noise model.
+    """

source/isaaclab/isaaclab/utils/noise/noise_model.py

@@ -189,3 +189,71 @@ def __call__(self, data: torch.Tensor) -> torch.Tensor:
             # now re-sample that expanded bias in-place
             self.reset()
         return super().__call__(data) + self._bias
+
+class ResetSampledNoiseModel(NoiseModel):
+    """Noise model that samples noise ONLY during reset and applies it consistently.
+
+    The noise is sampled from the configured distribution ONLY during reset and applied consistently
+    until the next reset. Unlike regular noise that generates new random values every step,
+    this model maintains the same noise values throughout an episode.
+    """
+
+    def __init__(self, noise_model_cfg: noise_cfg.NoiseModelCfg, num_envs: int, device: str):
+        # initialize parent class
+        super().__init__(noise_model_cfg, num_envs, device)
+        # store the noise configuration
+        self._noise_cfg = noise_model_cfg.noise_cfg
+        self._sampled_noise = torch.zeros((num_envs, 1), device=self._device)
+        self._num_components: int | None = None
+
+    def reset(self, env_ids: Sequence[int] | None = None):
+        """Reset the noise model by sampling NEW noise values.
+
+        This method samples new noise for the specified environments using the configured noise function.
+        The sampled noise will remain constant until the next reset.
+
+        Args:
+            env_ids: The environment ids to reset the noise model for. Defaults to None,
+                in which case all environments are considered.
+        """
+        # resolve the environment ids
+        if env_ids is None:
+            env_ids = slice(None)
+
+        # Use the existing noise function to sample new noise
+        # Create dummy data to sample from the noise function
+        dummy_data = torch.zeros((env_ids.stop - env_ids.start if isinstance(env_ids, slice) else len(env_ids), 1), 
+                                device=self._device)
+
+        # Sample noise using the configured noise function
+        sampled_noise = self._noise_model_cfg.noise_cfg.func(dummy_data, self._noise_model_cfg.noise_cfg)
+
+        self._sampled_noise[env_ids] = sampled_noise
+
+    def __call__(self, data: torch.Tensor) -> torch.Tensor:
+        """Apply the pre-sampled noise to the data.
+
+        This method applies the noise that was sampled during the last reset.
+        No new noise is generated - the same values are used consistently.
+
+        Args:
+            data: The data to apply the noise to. Shape is (num_envs, ...).
+
+        Returns:
+            The data with the noise applied. Shape is the same as the input data.
+        """
+        # on first apply, expand noise to match last dim of data
+        if self._num_components is None:
+            *_, self._num_components = data.shape
+            # expand noise from (num_envs,1) to (num_envs, num_components)
+            self._sampled_noise = self._sampled_noise.repeat(1, self._num_components)
+
+        # apply the noise based on operation
+        if self._noise_cfg.operation == "add":
+            return data + self._sampled_noise
+        elif self._noise_cfg.operation == "scale":
+            return data * self._sampled_noise
+        elif self._noise_cfg.operation == "abs":
+            return self._sampled_noise
+        else:
+            raise ValueError(f"Unknown operation in noise: {self._noise_cfg.operation}")

source/isaaclab_assets/isaaclab_assets/robots/universal_robots.py

@@ -55,47 +55,63 @@
 UR10e_CFG = ArticulationCfg(
     spawn=sim_utils.UsdFileCfg(
         usd_path=f"{ISAAC_NUCLEUS_DIR}/Robots/UniversalRobots/ur10e/ur10e.usd",
+        activate_contact_sensors=True,
         rigid_props=sim_utils.RigidBodyPropertiesCfg(
             disable_gravity=True,
             max_depenetration_velocity=5.0,
+            linear_damping=0.0,
+            angular_damping=0.0,
+            max_linear_velocity=1000.0,
+            max_angular_velocity=3666.0,
+            enable_gyroscopic_forces=True,
+            solver_position_iteration_count=192,
+            solver_velocity_iteration_count=1,
+            max_contact_impulse=1e32,
         ),
         articulation_props=sim_utils.ArticulationRootPropertiesCfg(
-            enabled_self_collisions=False, solver_position_iteration_count=16, solver_velocity_iteration_count=1
+            enabled_self_collisions=False, solver_position_iteration_count=192, solver_velocity_iteration_count=1
         ),
-        activate_contact_sensors=False,
+        collision_props=sim_utils.CollisionPropertiesCfg(contact_offset=0.005, rest_offset=0.0),
     ),
     init_state=ArticulationCfg.InitialStateCfg(
         joint_pos={
-            "shoulder_pan_joint": 3.141592653589793,
-            "shoulder_lift_joint": -1.5707963267948966,
-            "elbow_joint": 1.5707963267948966,
-            "wrist_1_joint": -1.5707963267948966,
-            "wrist_2_joint": -1.5707963267948966,
-            "wrist_3_joint": 0.0,
+            "shoulder_pan_joint": 2.7228e+00,
+            "shoulder_lift_joint": -8.3962e-01,
+            "elbow_joint": 1.3684e+00,
+            "wrist_1_joint": -2.1048e+00,
+            "wrist_2_joint": -1.5691e+00,
+            "wrist_3_joint": -1.9896e+00,
+            "finger_joint": 0.0,
         },
         pos=(0.0, 0.0, 0.0),
-        rot=(1.0, 0.0, 0.0, 0.0),
+        rot=(0.0, 0.0, 0.0, 1.0),
     ),
     actuators={
         # 'shoulder_pan_joint', 'shoulder_lift_joint', 'elbow_joint', 'wrist_1_joint', 'wrist_2_joint', 'wrist_3_joint'
         "shoulder": ImplicitActuatorCfg(
             joint_names_expr=["shoulder_.*"],
+            effort_limit=330.0,
+            velocity_limit=2.175,
             stiffness=1320.0,
-            damping=72.6636085,
+            damping=72.0,
             friction=0.0,
             armature=0.0,
         ),
         "elbow": ImplicitActuatorCfg(
             joint_names_expr=["elbow_joint"],
+            effort_limit=150.0,
+            velocity_limit=2.175,
             stiffness=600.0,
-            damping=34.64101615,
+            damping=50.0,
             friction=0.0,
             armature=0.0,
         ),
         "wrist": ImplicitActuatorCfg(
             joint_names_expr=["wrist_.*"],
+            effort_limit=56.0,
+            velocity_limit=2.175,
             stiffness=216.0,
-            damping=29.39387691,
+            damping=30.0,
             friction=0.0,
             armature=0.0,
         ),
@@ -132,34 +148,139 @@
 UR10e_ROBOTIQ_GRIPPER_CFG.init_state.joint_pos[".*_inner_finger_pad_joint"] = 0.0
 UR10e_ROBOTIQ_GRIPPER_CFG.init_state.joint_pos[".*_outer_.*_joint"] = 0.0
 # the major actuator joint for gripper
-UR10e_ROBOTIQ_GRIPPER_CFG.actuators["gripper_drive"] = ImplicitActuatorCfg(
+UR10e_ROBOTIQ_GRIPPER_CFG.actuators["finger_joint"] = ImplicitActuatorCfg(
     joint_names_expr=["finger_joint"],
-    effort_limit_sim=10.0,
-    velocity_limit_sim=1.0,
-    stiffness=11.25,
-    damping=0.1,
+    effort_limit_sim=1.0,
+    velocity_limit=2.175,
+    stiffness=20.0,
+    damping=8.94,
     friction=0.0,
-    armature=0.0,
+    armature=0.0,  # 0.57
 )
 # the auxiliary actuator joint for gripper
-UR10e_ROBOTIQ_GRIPPER_CFG.actuators["gripper_finger"] = ImplicitActuatorCfg(
-    joint_names_expr=[".*_inner_finger_joint"],
+UR10e_ROBOTIQ_GRIPPER_CFG.actuators["others_1"] = ImplicitActuatorCfg(
+    joint_names_expr=['right_outer_knuckle_joint', 'left_outer_finger_joint'],
     effort_limit_sim=1.0,
-    velocity_limit_sim=1.0,
-    stiffness=0.2,
-    damping=0.001,
+    velocity_limit=2.175,
+    stiffness=0.0,
+    damping=0.0,
     friction=0.0,
-    armature=0.0,
+    armature=0.0,  # 0.57
 )
 # the passive joints for gripper
-UR10e_ROBOTIQ_GRIPPER_CFG.actuators["gripper_passive"] = ImplicitActuatorCfg(
-    joint_names_expr=[".*_inner_finger_pad_joint", ".*_outer_finger_joint", "right_outer_knuckle_joint"],
+UR10e_ROBOTIQ_GRIPPER_CFG.actuators["others_2"] = ImplicitActuatorCfg(
+    joint_names_expr=['right_outer_finger_joint', 'left_inner_finger_joint', 'right_inner_finger_joint', 'left_inner_finger_pad_joint', 'right_inner_finger_pad_joint'],
     effort_limit_sim=1.0,
-    velocity_limit_sim=1.0,
-    stiffness=0.0,
-    damping=0.0,
+    velocity_limit=2.175,
+    stiffness=400.0,
+    damping=20.0,
     friction=0.0,
-    armature=0.0,
+    armature=0.0,  # 0.57
+)
+
+UR10e_2f140_CFG = ArticulationCfg(
+    spawn=sim_utils.UsdFileCfg(
+        usd_path=f"omniverse://isaac-dev.ov.nvidia.com/Projects/isaac_ros_gear_insertion/ur10e_robotiq_140_variant.usd",
+        # usd_path=f"omniverse://isaac-dev.ov.nvidia.com/Projects/IsaacARM/Assets/UR10/iakinola/ur10e_robotiq_140_variant.usd",
+        # usd_path=f"omniverse://isaac-dev.ov.nvidia.com/Isaac/Samples/Rigging/Manipulator/import_manipulator/ur10e/ur/ur_gripper.usd",
+        # rigid_props=sim_utils.RigidBodyPropertiesCfg(
+        #     disable_gravity=True,
+        #     max_depenetration_velocity=5.0,
+        # ),
+        activate_contact_sensors=True,
+        rigid_props=sim_utils.RigidBodyPropertiesCfg(
+            disable_gravity=True,
+            max_depenetration_velocity=5.0,
+            linear_damping=0.0,
+            angular_damping=0.0,
+            max_linear_velocity=1000.0,
+            max_angular_velocity=3666.0,
+            enable_gyroscopic_forces=True,
+            solver_position_iteration_count=192,
+            solver_velocity_iteration_count=1,
+            max_contact_impulse=1e32,
+        ),
+        articulation_props=sim_utils.ArticulationRootPropertiesCfg(
+            enabled_self_collisions=False, solver_position_iteration_count=192, solver_velocity_iteration_count=1
+        ),
+        collision_props=sim_utils.CollisionPropertiesCfg(contact_offset=0.005, rest_offset=0.0),
+        # articulation_props=sim_utils.ArticulationRootPropertiesCfg(
+        #         enabled_self_collisions=False, solver_position_iteration_count=12, solver_velocity_iteration_count=1
+        #     ),
+        # activate_contact_sensors=False,
+    ),    
+    init_state=ArticulationCfg.InitialStateCfg(
+            joint_pos={
+                "shoulder_pan_joint": 2.7228e+00,
+                "shoulder_lift_joint": -8.3962e-01,
+                "elbow_joint": 1.3684e+00,
+                "wrist_1_joint": -2.1048e+00,
+                "wrist_2_joint": -1.5691e+00,
+                "wrist_3_joint": -1.9896e+00,
+                "finger_joint": 0.0,
+            },
+            pos=(0.0, 0.0, 0.0),
+            # @ireti: Figure out why this rotaion is requried.
+            # Otherwise the IK during initialization is does not converge.
+            rot=(0.0, 0.0, 0.0, 1.0),
+        ),
+    actuators={
+            # @ireti: These values were obtained from 2025-05-30_20-35-06/params/env.yaml
+            "shoulder": ImplicitActuatorCfg(
+                joint_names_expr=["shoulder_.*"],
+                effort_limit=330.0,
+                velocity_limit=2.175,
+                stiffness=1320.0,
+                damping=72.0,
+                friction=0.0,
+                armature=0.0,
+            ),
+            "elbow": ImplicitActuatorCfg(
+                joint_names_expr=["elbow_joint"],
+                effort_limit=150.0,
+                velocity_limit=2.175,
+                stiffness=600.0,
+                damping=50.0,
+                friction=0.0,
+                armature=0.0,
+            ),
+            "wrist": ImplicitActuatorCfg(
+                joint_names_expr=["wrist_.*"],
+                effort_limit=56.0,
+                velocity_limit=2.175,
+                stiffness=216.0,
+                damping=30.0,
+                friction=0.0,
+                armature=0.0,
+            ),
+            "finger_joint": ImplicitActuatorCfg(
+                joint_names_expr=["finger_joint"],
+                effort_limit_sim=1.0,
+                velocity_limit=2.175,
+                stiffness=20.0,
+                damping=8.94,
+                friction=0.0,
+                armature=0.0,  # 0.57
+            ),
+            "others_1": ImplicitActuatorCfg(
+                joint_names_expr=['right_outer_knuckle_joint', 'left_outer_finger_joint'],
+                effort_limit_sim=1.0,
+                velocity_limit=2.175,
+                stiffness=0.0,
+                damping=0.0,
+                friction=0.0,
+                armature=0.0,  # 0.57
+            ),
+            "others_2": ImplicitActuatorCfg(
+                joint_names_expr=['right_outer_finger_joint', 'left_inner_finger_joint', 'right_inner_finger_joint', 'left_inner_finger_pad_joint', 'right_inner_finger_pad_joint'],
+                effort_limit_sim=1.0,
+                velocity_limit=2.175,
+                stiffness=400.0,
+                damping=20.0,
+                friction=0.0,
+                armature=0.0,  # 0.57
+            ),
+    }
 )
 
 """Configuration of UR-10E arm with Robotiq_2f_140 gripper."""

source/isaaclab_rl/isaaclab_rl/rsl_rl/rl_cfg.py

@@ -31,6 +31,9 @@ class RslRlPpoActorCriticCfg:
     noise_std_type: Literal["scalar", "log"] = "scalar"
     """The type of noise standard deviation for the policy. Default is scalar."""
 
+    state_dependent_std: bool = False
+    """Whether to use state-dependent standard deviation for the policy. Default is False."""
+
     actor_obs_normalization: bool = MISSING
     """Whether to normalize the observation for the actor network."""
 

source/isaaclab_rl/setup.py

@@ -46,11 +46,10 @@
         "rl-games @ git+https://github.com/isaac-sim/[email protected]",
         "gym",
     ],  # rl-games still needs gym :(
-    "rsl-rl": ["rsl-rl-lib==3.0.1"],
+    "rsl-rl": ["rsl-rl-lib==3.1.0"],
 }
 # Add the names with hyphens as aliases for convenience
 EXTRAS_REQUIRE["rl_games"] = EXTRAS_REQUIRE["rl-games"]
-EXTRAS_REQUIRE["rsl_rl"] = EXTRAS_REQUIRE["rsl-rl"]
 
 # Cumulation of all extra-requires
 EXTRAS_REQUIRE["all"] = list(itertools.chain.from_iterable(EXTRAS_REQUIRE.values()))

source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/deploy/gear_assembly/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) 2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Assemble 3 gears into a base."""

source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/deploy/gear_assembly/config/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Configurations for arm-based gear assembly environments."""
+
+# We leave this file empty since we don't want to expose any configs in this package directly.
+# We still need this file to import the "config" module in the parent package.