The timestamps of each stream do not match.

[wataru-okumura]

I always enjoy using Orbbec FemtoBolt.
I have a question I would like to know about using OrbbecSDK_ROS2.

script.py is a program that, when executed, retrieves a timestamp for each stream image.
However, the timestamps of each stream do not match.
Is there any way to get the same timestamp data for each stream by using ROS parameters etc. or by modifying the SDK code?

Thank you in advance.

Ubuntu 20.04.6 LTS
ROS2

terminal1

ros2 launch orbbec_camera femto_bolt.launch.py

terminal2

python script.py

script.py

import rclpy
from rclpy.node import Node
from sensor_msgs.msg import Image
from sensor_msgs.msg import CameraInfo

class OrbbecWrapperNode():

    def __init__(self, mod_ns: str):
        self._mod_ns = mod_ns
        rclpy.init(args=None)
        self.node = rclpy.create_node('orbbec_' + mod_ns, use_global_arguments=False)

        # Camera related subscribers
        self.camera_color_info = CameraInfo()
        self.camera_color_image = Image()
        self.camera_depth_info = CameraInfo()
        self.camera_depth_image = Image()
        self.camera_depth_points = None
        self.camera_gyro_accel = None
        self.camera_ir_info = CameraInfo()
        self.camera_ir_image = Image()
        self.tf_data = None
        self.tf_static_data = None

        # Subscribers
        self.col_img_sub = self.node.create_subscription(
            Image,
            '/' + mod_ns + '/color/image_raw',
            self._camera_color_image_callback,
            10
        )

        self.depth_img_sub = self.node.create_subscription(
            Image,
            '/' + mod_ns + '/depth/image_raw',
            self._camera_depth_image_callback,
            10
        )

        self.ir_img_sub = self.node.create_subscription(
            Image,
            '/' + mod_ns + '/ir/image_raw',
            self._camera_ir_image_callback,
            10
        )

    def _camera_color_image_callback(self, msg):
        timestamp_str = "Color Image Timestamp: " + str(msg.header.stamp.sec) + "." + str(msg.header.stamp.nanosec)
        self.node.get_logger().info(timestamp_str)

    def _camera_depth_image_callback(self, msg):
        timestamp_str = "Depth Image Timestamp: " + str(msg.header.stamp.sec) + "." + str(msg.header.stamp.nanosec)
        self.node.get_logger().info(timestamp_str)

    def _camera_ir_image_callback(self, msg):
        timestamp_str = "IR Image Timestamp: " + str(msg.header.stamp.sec) + "." + str(msg.header.stamp.nanosec)
        self.node.get_logger().info(timestamp_str)

def main(args=None):
    orbbec_wrapper = OrbbecWrapperNode("camera")

    while rclpy.ok():
        rclpy.spin_once(orbbec_wrapper.node)

    orbbec_wrapper.node.destroy_node()
    rclpy.shutdown()

if __name__ == '__main__':
    main()

terminal

INFO] [1717485408.883511720] [orbbec_camera]: Depth Image Timestamp: 1717485408.848007936
[INFO] [1717485408.884886989] [orbbec_camera]: IR Image Timestamp: 1717485408.848007936
[INFO] [1717485408.889212137] [orbbec_camera]: Color Image Timestamp: 1717485408.853852928
[INFO] [1717485408.890679757] [orbbec_camera]: Depth Image Timestamp: 1717485408.878413056
[INFO] [1717485408.891224520] [orbbec_camera]: IR Image Timestamp: 1717485408.878413056
[INFO] [1717485408.896047447] [orbbec_camera]: Color Image Timestamp: 1717485408.887087104
[INFO] [1717485408.922920147] [orbbec_camera]: Depth Image Timestamp: 1717485408.914636032
[INFO] [1717485408.923664492] [orbbec_camera]: IR Image Timestamp: 1717485408.914636032
[INFO] [1717485408.929931520] [orbbec_camera]: Color Image Timestamp: 1717485408.920285952
[INFO] [1717485408.956523193] [orbbec_camera]: Depth Image Timestamp: 1717485408.947889920
[INFO] [1717485408.957308215] [orbbec_camera]: IR Image Timestamp: 1717485408.947889920
[INFO] [1717485408.963221301] [orbbec_camera]: Color Image Timestamp: 1717485408.953464064
[INFO] [1717485408.989244972] [orbbec_camera]: Depth Image Timestamp: 1717485408.978404096
[INFO] [1717485408.990323433] [orbbec_camera]: IR Image Timestamp: 1717485408.978404096
[INFO] [1717485408.997314429] [orbbec_camera]: Color Image Timestamp: 1717485408.986700032
[INFO] [1717485409.022324539] [orbbec_camera]: Depth Image Timestamp: 1717485409.13319936
[INFO] [1717485409.023408473] [orbbec_camera]: IR Image Timestamp: 1717485409.13319936
[INFO] [1717485409.030325845] [orbbec_camera]: Color Image Timestamp: 1717485409.19878912
[INFO] [1717485409.056319313] [orbbec_camera]: Depth Image Timestamp: 1717485409.47595008
[INFO] [1717485409.057421909] [orbbec_camera]: IR Image Timestamp: 1717485409.47595008
[INFO] [1717485409.064464855] [orbbec_camera]: Color Image Timestamp: 1717485409.53082112
[INFO] [1717485409.088765863] [orbbec_camera]: Depth Image Timestamp: 1717485409.77477888
[INFO] [1717485409.089855691] [orbbec_camera]: IR Image Timestamp: 1717485409.77477888
[INFO] [1717485409.097024796] [orbbec_camera]: Color Image Timestamp: 1717485409.86302976
[INFO] [1717485409.122985457] [orbbec_camera]: Depth Image Timestamp: 1717485409.114063872
[INFO] [1717485409.124057076] [orbbec_camera]: IR Image Timestamp: 1717485409.114063872
[INFO] [1717485409.131338987] [orbbec_camera]: Color Image Timestamp: 1717485409.119472128

[wataru-okumura]

There was some information omitted from the above entry.
I am using Release v1.5.5.

[jian-dong]

I have fixed the configuration of the femto bolt and checked it with the following script. The timestamp difference between depth and color is around 1ms:

#!/usr/bin/env python3

import rclpy
from rclpy.node import Node
from cv_bridge import CvBridge

from sensor_msgs.msg import Image
from std_msgs.msg import Float64

from message_filters import ApproximateTimeSynchronizer, Subscriber

class OpenVINOSegROS(Node):
    def __init__(self):
        super().__init__('openvino_segmentation_node')
        self.bridge = CvBridge()
        self.color_sub = Subscriber(self, Image, '/camera/color/image_raw')
        self.depth_sub = Subscriber(self, Image, '/camera/depth/image_raw')
        self.time_diff_pub = self.create_publisher(Float64, '/sync_time_diff', 10)

        self.ts = ApproximateTimeSynchronizer([self.color_sub, self.depth_sub], 10, 0.1)
        self.ts.registerCallback(self.image_callback)

    def image_callback(self, rosmsg_color_image, rosmsg_depth_image):
        color_timestamp = rosmsg_color_image.header.stamp
        depth_timestamp = rosmsg_depth_image.header.stamp
        time_diff = (depth_timestamp.sec - color_timestamp.sec) * 1000 + (depth_timestamp.nanosec - color_timestamp.nanosec) / 1e6
        self.get_logger().info(f"time diff(ms): {time_diff:.3f}")

        # Publish the time difference
        time_diff_msg = Float64()
        time_diff_msg.data = time_diff
        self.time_diff_pub.publish(time_diff_msg)

    def run(self):
        rclpy.spin(self)

def main(args=None):
    rclpy.init(args=args)
    seg_detector = OpenVINOSegROS()
    seg_detector.run()
    rclpy.shutdown()

if __name__ == "__main__":
    main()

[wataru-okumura]

Sorry for the late reply.
I could indeed confirm that color and depth are timestamps with intervals of a few ms.
Thank you very much for your help.

Personally, it would be great if we could receive image data using TimeSynchronizer instead of AproximateTimeSynchronizer, but upgraded to deliver color and depth at 0ms intervals (i.e. with perfectly synchronized timestamps). Is there any plan to do this?