feat(autoware_tensorrt_bevdet): add new 3d object detection method

build_depends.repos

@@ -58,3 +58,7 @@ repositories:
     type: git
     url: https://github.com/autowarefoundation/ament_cmake.git
     version: feat/faster_ament_libraries_deduplicate
+  universe/external/bevdet_vendor:
+    type: git
+    url: https://github.com/autowarefoundation/bevdet_vendor.git
+    version: bevdet_vendor-ros2

launch/tier4_perception_launch/launch/object_recognition/detection/detection.launch.xml

@@ -221,6 +221,28 @@
       <arg name="radar_object_clustering_param_path" value="$(var radar_object_clustering_param_path)"/>
     </include>
   </group>
+  <group if="$(eval '&quot;$(var mode)&quot;==&quot;camera&quot;')">
+    <!-- Camera-based bev detectors-->
+    <include file="$(find-pkg-share tier4_perception_launch)/launch/object_recognition/detection/detector/camera_bev_detector.launch.xml">
+      <arg name="input/camera0/image" value="$(var input/camera0/image)"/>
+      <arg name="input/camera0/info" value="$(var input/camera0/info)"/>
+      <arg name="input/camera1/image" value="$(var input/camera1/image)"/>
+      <arg name="input/camera1/info" value="$(var input/camera1/info)"/>
+      <arg name="input/camera2/image" value="$(var input/camera2/image)"/>
+      <arg name="input/camera2/info" value="$(var input/camera2/info)"/>
+      <arg name="input/camera3/image" value="$(var input/camera3/image)"/>
+      <arg name="input/camera3/info" value="$(var input/camera3/info)"/>
+      <arg name="input/camera4/image" value="$(var input/camera4/image)"/>
+      <arg name="input/camera4/info" value="$(var input/camera4/info)"/>
+      <arg name="input/camera5/image" value="$(var input/camera5/image)"/>
+      <arg name="input/camera5/info" value="$(var input/camera5/info)"/>
+      <arg name="input/camera6/image" value="$(var input/camera6/image)"/>
+      <arg name="input/camera6/info" value="$(var input/camera6/info)"/>
+      <arg name="input/camera7/image" value="$(var input/camera7/image)"/>
+      <arg name="input/camera7/info" value="$(var input/camera7/info)"/>
+      <arg name="number_of_cameras" value="$(var number_of_cameras)"/>
+    </include>
+  </group>
   <!-- DetectionByTracker -->
   <group if="$(var switch/detector/tracker_based)">
     <include file="$(find-pkg-share tier4_perception_launch)/launch/object_recognition/detection/detector/tracker_based_detector.launch.xml">

launch/tier4_perception_launch/launch/object_recognition/detection/detector/camera_bev_detector.launch.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0"?>
+<launch>
+  <!-- Camera parameters -->
+  <arg name="input/camera0/image"/>
+  <arg name="input/camera0/info"/>
+  <arg name="input/camera1/image"/>
+  <arg name="input/camera1/info"/>
+  <arg name="input/camera2/image"/>
+  <arg name="input/camera2/info"/>
+  <arg name="input/camera3/image"/>
+  <arg name="input/camera3/info"/>
+  <arg name="input/camera4/image"/>
+  <arg name="input/camera4/info"/>
+  <arg name="input/camera5/image"/>
+  <arg name="input/camera5/info"/>
+  <arg name="input/camera6/image"/>
+  <arg name="input/camera6/info"/>
+  <arg name="input/camera7/image"/>
+  <arg name="input/camera7/info"/>
+  <arg name="number_of_cameras"/>
+
+  <!-- Model parameters -->
+  <arg name="data_path" default="$(env HOME)/autoware_data" description="packages data and artifacts directory path"/>
+  <arg name="bevdet_model_name" default="bevdet_one_lt_d"/>
+  <arg name="bevdet_model_path" default="$(var data_path)/tensorrt_bevdet"/>
+  <!-- BEVDet -->
+  <group>
+    <push-ros-namespace namespace="bevdet"/>
+    <include file="$(find-pkg-share autoware_tensorrt_bevdet)/launch/tensorrt_bevdet.launch.xml">
+      <arg name="input/img_fl" value="$(var input/camera0/image)"/>
+      <arg name="input/img_fl/camera_info" value="$(var input/camera0/info)"/>
+      <arg name="input/img_f" value="$(var input/camera1/image)"/>
+      <arg name="input/img_f/camera_info" value="$(var input/camera1/info)"/>
+      <arg name="input/img_fr" value="$(var input/camera2/image)"/>
+      <arg name="input/img_fr/camera_info" value="$(var input/camera2/info)"/>
+      <arg name="input/img_bl" value="$(var input/camera3/image)"/>
+      <arg name="input/img_bl/camera_info" value="$(var input/camera3/info)"/>
+      <arg name="input/img_b" value="$(var input/camera4/image)"/>
+      <arg name="input/img_b/camera_info" value="$(var input/camera4/info)"/>
+      <arg name="input/img_br" value="$(var input/camera5/image)"/>
+      <arg name="input/img_br/camera_info" value="$(var input/camera5/info)"/>
+      <arg name="output/boxes" value="boxes"/>
+      <arg name="data_path" value="$(var data_path)"/>
+      <arg name="model_name" value="$(var bevdet_model_name)"/>
+      <arg name="model_path" value="$(var bevdet_model_path)"/>
+    </include>
+  </group>
+</launch>

perception/autoware_tensorrt_bevdet/CMakeLists.txt

@@ -0,0 +1,52 @@
+cmake_minimum_required(VERSION 3.17)
+project(autoware_tensorrt_bevdet)
+
+add_compile_options(-std=c++17)
+
+find_package(bevdet_vendor)
+if(NOT ${bevdet_vendor_FOUND})
+  message(WARNING "The bevdet_vendor package is not found. Please check its dependencies.")
+  return()
+endif()
+
+find_package(autoware_cmake REQUIRED)
+autoware_package()
+find_package(rclcpp REQUIRED)
+find_package(rclcpp_components REQUIRED)
+find_package(Eigen3 REQUIRED)
+find_package(sensor_msgs REQUIRED)
+find_package(cv_bridge REQUIRED)
+find_package(tf2_geometry_msgs REQUIRED)
+find_package(autoware_perception_msgs REQUIRED)
+
+ament_auto_add_library(${PROJECT_NAME}_component SHARED
+    src/bevdet_node.cpp
+    src/ros_utils.cpp
+)
+
+include_directories(
+  include
+  ${EIGEN3_INCLUDE_DIRS}
+  ${OpenCV_INCLUDE_DIRS}
+  ${cv_bridge_INCLUDE_DIRS}
+  ${tf2_geometry_msgs_INCLUDE_DIRS}
+  ${autoware_perception_msgs_INCLUDE_DIRS}
+  ${bevdet_vendor_INCLUDE_DIRS}
+)
+
+target_link_libraries(${PROJECT_NAME}_component
+    rclcpp::rclcpp
+    ${bevdet_vendor_TARGETS}
+    ${OpenCV_LIBS}
+)
+
+rclcpp_components_register_node(${PROJECT_NAME}_component
+    PLUGIN "autoware::tensorrt_bevdet::TRTBEVDetNode"
+    EXECUTABLE ${PROJECT_NAME}_node
+)
+
+ament_auto_package(
+  INSTALL_TO_SHARE
+    launch
+    config
+)

perception/autoware_tensorrt_bevdet/README.md

@@ -0,0 +1,77 @@
+# tensorrt_bevdet <!-- cspell: ignore bevdet -->
+
+## Purpose
+
+The core algorithm, named `BEVDet`, it unifies multi-view images into the perspective of BEV for 3D object detection task.
+
+## Inner-workings / Algorithms
+
+### Cite
+
+<!-- cspell: ignore Junjie Huang, Guan Huang -->
+
+- Junjie Huang, Guan Huang, "BEVPoolv2: A Cutting-edge Implementation of BEVDet Toward Deployment", [[ref](https://arxiv.org/pdf/2211.17111)]
+- [bevdet_vendor](https://github.com/autowarefoundation/bevdet_vendor) package are copied from the [original codes](https://github.com/LCH1238/bevdet-tensorrt-cpp/tree/one) (The TensorRT, C++ implementation by LCH1238) and modified.
+- This package is ported version toward Autoware from [bevdet_vendor](https://github.com/autowarefoundation/bevdet_vendor).
+
+## Inputs / Outputs
+
+### Inputs
+
+| Name                               | Type                           | Description                         |
+| ---------------------------------- | ------------------------------ | ----------------------------------- |
+| `~/input/topic_img_fl`             | `sensor_msgs::msg::Image`      | input front_left camera image       |
+| `~/input/topic_img_f`              | `sensor_msgs::msg::Image`      | input front camera image            |
+| `~/input/topic_img_fr`             | `sensor_msgs::msg::Image`      | input front_right camera image      |
+| `~/input/topic_img_bl`             | `sensor_msgs::msg::Image`      | input back_left camera image        |
+| `~/input/topic_img_b`              | `sensor_msgs::msg::Image`      | input back camera image             |
+| `~/input/topic_img_br`             | `sensor_msgs::msg::Image`      | input back_right camera image       |
+| `~/input/topic_img_fl/camera_info` | `sensor_msgs::msg::CameraInfo` | input front_left camera parameters  |
+| `~/input/topic_img_f/camera_info`  | `sensor_msgs::msg::CameraInfo` | input front camera parameters       |
+| `~/input/topic_img_fr/camera_info` | `sensor_msgs::msg::CameraInfo` | input front_right camera parameters |
+| `~/input/topic_img_bl/camera_info` | `sensor_msgs::msg::CameraInfo` | input back_left camera parameters   |
+| `~/input/topic_img_b/camera_info`  | `sensor_msgs::msg::CameraInfo` | input back camera parameters        |
+| `~/input/topic_img_br/camera_info` | `sensor_msgs::msg::CameraInfo` | input back_right camera parameters  |
+
+### Outputs
+
+| Name             | Type                                             | Description      |
+| ---------------- | ------------------------------------------------ | ---------------- |
+| `~/output/boxes` | `autoware_perception_msgs::msg::DetectedObjects` | detected objects |
+
+## How to Use Tensorrt BEVDet Node
+
+1. launch `tensorrt_bevdet_node`
+
+   ```bash
+
+   ros2 launch autoware_tensorrt_bevdet tensorrt_bevdet_node.launch.xml
+   ```
+
+2. play ros2 bag of nuScenes data
+
+   Please refer to open source repository [ros2_dataset_bridge](https://github.com/Owen-Liuyuxuan/ros2_dataset_bridge) to publish the ROS 2 topics for NuScenes dataset.
+
+## Trained Models
+
+You can download the onnx format of trained models by clicking on the links below.
+
+- BEVDet: [bevdet_one_lt_d.onnx](https://drive.google.com/file/d/1eMGJfdCVlDPBphBTjMcnIh3wdW7Q7WZB/view?usp=sharing)
+
+The `BEVDet` model was trained in `NuScenes` dataset for 20 epochs.
+
+## Limitation
+
+The model is trained on open-source dataset `NuScenes` and has poor generalization on its own dataset, If you want to use this model to infer your data, you need to retrain it.
+
+## Training BEVDet Model
+
+If you want to train model using the [TIER IV's internal database(~2600 key frames)](https://drive.google.com/file/d/1UaarK88HZu09sf7Ix-bEVl9zGNGFwTVL/view?usp=sharing), please refer to the following repositories:[BEVDet adapted to TIER IV dataset](https://github.com/cyn-liu/BEVDet/tree/train_export).
+
+## References/External links
+
+[1] <https://github.com/HuangJunJie2017/BEVDet/tree/dev2.1>
+
+[2] <https://github.com/LCH1238/BEVDet/tree/export>
+
+[3] <https://github.com/LCH1238/bevdet-tensorrt-cpp/tree/one>

perception/autoware_tensorrt_bevdet/config/bevdet.param.yaml

@@ -0,0 +1,12 @@
+/**:
+    ros__parameters:
+      # weight files
+      onnx_path: "$(var model_path)/$(var model_name).onnx"
+      engine_path: "$(var model_path)/$(var model_name).engine"
+      data_params:
+        CAM_NUM: 6
+        cams: ["CAM_FRONT_LEFT", "CAM_FRONT", "CAM_FRONT_RIGHT", "CAM_BACK_LEFT", "CAM_BACK", "CAM_BACK_RIGHT"]
+      post_process_params:
+        # post-process params
+        score_threshold: 0.2
+        class_names: ["car", "truck", "construction_vehicle", "bus", "trailer", "barrier", "motorcycle", "bicycle", "pedestrian", "traffic_cone"]

perception/autoware_tensorrt_bevdet/config/bevdet_r50_4dlongterm_depth.yaml

@@ -0,0 +1,63 @@
+bev_range: [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0]
+mean: [123.675, 116.28, 103.53]
+std: [58.395, 57.12, 57.375]
+use_depth: true
+use_adj: true
+adj_num: 8
+transform_size: 6
+cam_params_size: 27
+sampling: nearest # nearest # or bicubic
+data_config:
+  Ncams: 6
+  cams: [CAM_FRONT_LEFT, CAM_FRONT, CAM_FRONT_RIGHT, CAM_BACK_LEFT, CAM_BACK, CAM_BACK_RIGHT]
+  resize_radio: 0.44
+  crop: [140, 0]
+  flip: true
+  input_size: [256, 704]
+  resize: [-0.06, 0.11]
+  resize_test: 0.0
+  rot: [-5.4, 5.4]
+  src_size: [900, 1600]
+grid_config:
+  depth: [1.0, 60.0, 0.5]
+  x: [-51.2, 51.2, 0.8]
+  y: [-51.2, 51.2, 0.8]
+  z: [-5, 3, 8]
+model:
+  bevpool_channels: 80
+  coder:
+    code_size: 9
+    max_num: 500
+    post_center_range: [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0]
+    score_threshold: 0.1
+  common_head:
+    channels: [2, 1, 3, 2, 2]
+    names: [reg, height, dim, rot, vel]
+  down_sample: 16
+  tasks:
+    - class_names:
+        [
+          car,
+          truck,
+          construction_vehicle,
+          bus,
+          trailer,
+          barrier,
+          motorcycle,
+          bicycle,
+          pedestrian,
+          traffic_cone,
+        ]
+      num_class: 10
+test_cfg:
+  max_per_img: 500
+  max_pool_nms: false
+  min_radius: [4, 12, 10, 1, 0.85, 0.175]
+  nms_rescale_factor:
+    - [1.0, 0.7, 0.7, 0.4, 0.55, 1.1, 1.0, 1.0, 1.5, 3.5]
+  nms_thr: [0.2]
+  nms_type: [rotate]
+  post_center_limit_range: [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0]
+  post_max_size: 500
+  pre_max_size: 1000
+  score_threshold: 0.1