feat(autoware_tensorrt_bevdet): add new 3d object detection method

launch/tier4_perception_launch/launch/object_recognition/detection/detection.launch.xml

@@ -211,6 +211,28 @@
       <arg name="radar_object_clustering_param_path" value="$(var radar_object_clustering_param_path)"/>
     </include>
   </group>
+  <group if="$(eval '&quot;$(var mode)&quot;==&quot;camera&quot;')">
+    <!-- Camera-based bev detectors-->
+    <include file="$(find-pkg-share tier4_perception_launch)/launch/object_recognition/detection/detector/camera_bev_detector.launch.xml">
+      <arg name="input/camera0/image" value="$(var input/camera0/image)"/>
+      <arg name="input/camera0/info" value="$(var input/camera0/info)"/>
+      <arg name="input/camera1/image" value="$(var input/camera1/image)"/>
+      <arg name="input/camera1/info" value="$(var input/camera1/info)"/>
+      <arg name="input/camera2/image" value="$(var input/camera2/image)"/>
+      <arg name="input/camera2/info" value="$(var input/camera2/info)"/>
+      <arg name="input/camera3/image" value="$(var input/camera3/image)"/>
+      <arg name="input/camera3/info" value="$(var input/camera3/info)"/>
+      <arg name="input/camera4/image" value="$(var input/camera4/image)"/>
+      <arg name="input/camera4/info" value="$(var input/camera4/info)"/>
+      <arg name="input/camera5/image" value="$(var input/camera5/image)"/>
+      <arg name="input/camera5/info" value="$(var input/camera5/info)"/>
+      <arg name="input/camera6/image" value="$(var input/camera6/image)"/>
+      <arg name="input/camera6/info" value="$(var input/camera6/info)"/>
+      <arg name="input/camera7/image" value="$(var input/camera7/image)"/>
+      <arg name="input/camera7/info" value="$(var input/camera7/info)"/>
+      <arg name="number_of_cameras" value="$(var number_of_cameras)"/>
+    </include>
+  </group>
   <!-- DetectionByTracker -->
   <group if="$(var switch/detector/tracker_based)">
     <include file="$(find-pkg-share tier4_perception_launch)/launch/object_recognition/detection/detector/tracker_based_detector.launch.xml">

launch/tier4_perception_launch/launch/object_recognition/detection/detector/camera_bev_detector.launch.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0"?>
+<launch>
+  <!-- Camera parameters -->
+  <arg name="input/camera0/image"/>
+  <arg name="input/camera0/info"/>
+  <arg name="input/camera1/image"/>
+  <arg name="input/camera1/info"/>
+  <arg name="input/camera2/image"/>
+  <arg name="input/camera2/info"/>
+  <arg name="input/camera3/image"/>
+  <arg name="input/camera3/info"/>
+  <arg name="input/camera4/image"/>
+  <arg name="input/camera4/info"/>
+  <arg name="input/camera5/image"/>
+  <arg name="input/camera5/info"/>
+  <arg name="input/camera6/image"/>
+  <arg name="input/camera6/info"/>
+  <arg name="input/camera7/image"/>
+  <arg name="input/camera7/info"/>
+  <arg name="number_of_cameras"/>
+
+  <!-- Model parameters -->
+  <arg name="data_path" default="$(env HOME)/autoware_data" description="packages data and artifacts directory path"/>
+  <arg name="bevdet_model_name" default="bevdet_one_lt_d"/>
+  <arg name="bevdet_model_path" default="$(var data_path)/tensorrt_bevdet"/>
+  <!-- BEVDet -->
+  <group>
+    <push-ros-namespace namespace="bevdet"/>
+    <include file="$(find-pkg-share autoware_tensorrt_bevdet)/launch/tensorrt_bevdet.launch.xml">
+      <arg name="input/img_fl" value="$(var input/camera0/image)"/>
+      <arg name="input/img_fl/camera_info" value="$(var input/camera0/info)"/>
+      <arg name="input/img_f" value="$(var input/camera1/image)"/>
+      <arg name="input/img_f/camera_info" value="$(var input/camera1/info)"/>
+      <arg name="input/img_fr" value="$(var input/camera2/image)"/>
+      <arg name="input/img_fr/camera_info" value="$(var input/camera2/info)"/>
+      <arg name="input/img_bl" value="$(var input/camera3/image)"/>
+      <arg name="input/img_bl/camera_info" value="$(var input/camera3/info)"/>
+      <arg name="input/img_b" value="$(var input/camera4/image)"/>
+      <arg name="input/img_b/camera_info" value="$(var input/camera4/info)"/>
+      <arg name="input/img_br" value="$(var input/camera5/image)"/>
+      <arg name="input/img_br/camera_info" value="$(var input/camera5/info)"/>
+      <arg name="output/boxes" value="boxes"/>
+      <arg name="data_path" value="$(var data_path)"/>
+      <arg name="model_name" value="$(var bevdet_model_name)"/>
+      <arg name="model_path" value="$(var bevdet_model_path)"/>
+    </include>
+  </group>
+</launch>

perception/autoware_tensorrt_bevdet/CMakeLists.txt

@@ -0,0 +1,102 @@
+cmake_minimum_required(VERSION 3.17)
+project(autoware_tensorrt_bevdet)
+
+add_compile_options(-W)
+add_compile_options(-std=c++17)
+set(CMAKE_CXX_FLAGS_RELEASE "-Wno-deprecated-declarations -O2")
+set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++17 -O0 -Xcompiler -fPIC -g -w -gencode=arch=compute_86,code=sm_86")
+find_package(tensorrt_common)
+if(NOT ${tensorrt_common_FOUND})
+  message(WARNING "The tensorrt_common package is not found. Please check its dependencies.")
+  return()
+endif()
+
+find_package(ament_cmake REQUIRED)
+find_package(cudnn_cmake_module REQUIRED)
+find_package(tensorrt_cmake_module REQUIRED)
+find_package(rclcpp REQUIRED)
+find_package(yaml-cpp REQUIRED)
+find_package(Eigen3 REQUIRED)
+find_package(OpenCV REQUIRED)
+find_package(PCL REQUIRED)
+find_package(sensor_msgs REQUIRED)
+find_package(pcl_conversions REQUIRED)
+find_package(cv_bridge REQUIRED)
+find_package(tf2_geometry_msgs REQUIRED)
+find_package(autoware_perception_msgs REQUIRED)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit)
+find_package(CUDNN)
+find_package(TENSORRT)
+if(NOT (CUDAToolkit_FOUND AND CUDNN_FOUND AND TENSORRT_FOUND))
+  message(WARNING "cuda, cudnn, tensorrt libraries are not found")
+  return()
+endif()
+
+include_directories(
+  include
+  SYSTEM
+  ${YAML_CPP_INCLUDE_DIRS}
+  ${EIGEN3_INCLUDE_DIRS}
+  ${OpenCV_INCLUDE_DIRS}
+  ${PCL_INCLUDE_DIRS}
+  ${cv_bridge_INCLUDE_DIRS}
+  ${tf2_geometry_msgs_INCLUDE_DIRS}
+  ${autoware_perception_msgs_INCLUDE_DIRS}
+)
+
+cuda_add_executable(${PROJECT_NAME}_node
+    src/bevdet_node.cpp
+    src/bevdet.cpp
+    src/preprocess.cu
+    src/iou3d_nms.cu
+    src/postprocess.cu
+    src/data.cpp
+    src/cpu_jpegdecoder.cpp
+    src/nvjpegdecoder.cpp
+
+    src/preprocess_plugin.cu
+    src/bevpool_plugin.cu
+    src/alignbev_plugin.cu
+    src/gatherbev_plugin.cu
+)
+
+# Link libraries to executable
+ament_target_dependencies(${PROJECT_NAME}_node
+  "rclcpp"
+  "sensor_msgs"
+  "pcl_conversions"
+  "cv_bridge"
+  "autoware_perception_msgs"
+  "tf2_geometry_msgs"
+)
+
+target_link_libraries(${PROJECT_NAME}_node
+    yaml-cpp
+    libnvinfer.so
+    libnvonnxparser.so
+    libz.so
+    libjpeg.so
+    rclcpp::rclcpp
+    stdc++fs
+    ${NVINFER}
+    ${TENSORRT_LIBRARIES}
+    ${CUDA_LIBRARIES}
+    ${CUBLAS_LIBRARIES}
+    ${CUDNN_LIBRARY}
+    ${OpenCV_LIBS}
+    ${PCL_LIBRARIES}
+)
+
+install(TARGETS ${PROJECT_NAME}_node
+  DESTINATION lib/${PROJECT_NAME}
+)
+
+install(DIRECTORY
+  launch
+  config
+  DESTINATION share/${PROJECT_NAME}/
+)
+
+ament_package()

perception/autoware_tensorrt_bevdet/README.md

@@ -0,0 +1,56 @@
+# tensorrt_bevdet <!-- cspell: ignore bevdet -->
+
+## Purpose
+
+tensorrt_bevdet is a dynamic 3D bev object detection package based on 6 surround view cameras.
+
+## Inner-workings / Algorithms
+
+BEVDet is a BEV perception algorithm based on panoramic cameras. It unifies multi-view images into the perspective of BEV for 3D object detection task. In this implementation, BEVDet network to inference with TensorRT.
+
+## Inputs / Outputs
+
+### Inputs
+
+| Name                               | Type                           | Description                         |
+| ---------------------------------- | ------------------------------ | ----------------------------------- |
+| `~/input/topic_img_fl`             | `sensor_msgs::msg::Image`      | input front_left camera image       |
+| `~/input/topic_img_f`              | `sensor_msgs::msg::Image`      | input front camera image            |
+| `~/input/topic_img_fr`             | `sensor_msgs::msg::Image`      | input front_right camera image      |
+| `~/input/topic_img_bl`             | `sensor_msgs::msg::Image`      | input back_left camera image        |
+| `~/input/topic_img_b`              | `sensor_msgs::msg::Image`      | input back camera image             |
+| `~/input/topic_img_br`             | `sensor_msgs::msg::Image`      | input back_right camera image       |
+| `~/input/topic_img_fl/camera_info` | `sensor_msgs::msg::CameraInfo` | input front_left camera parameters  |
+| `~/input/topic_img_f/camera_info`  | `sensor_msgs::msg::CameraInfo` | input front camera parameters       |
+| `~/input/topic_img_fr/camera_info` | `sensor_msgs::msg::CameraInfo` | input front_right camera parameters |
+| `~/input/topic_img_bl/camera_info` | `sensor_msgs::msg::CameraInfo` | input back_left camera parameters   |
+| `~/input/topic_img_b/camera_info`  | `sensor_msgs::msg::CameraInfo` | input back camera parameters        |
+| `~/input/topic_img_br/camera_info` | `sensor_msgs::msg::CameraInfo` | input back_right camera parameters  |
+
+### Outputs
+
+| Name             | Type                                             | Description      |
+| ---------------- | ------------------------------------------------ | ---------------- |
+| `~/output/boxes` | `autoware_perception_msgs::msg::DetectedObjects` | detected objects |
+
+## Limitation
+
+The model is trained on open-source dataset `NuScenes` and has poor generalization on its own dataset, If you want to use this model to infer your data, you need to retrain it.
+
+## Trained Models
+
+You can download the onnx format of trained models by clicking on the links below.
+
+- BEVDet: [bevdet_one_lt_d.onnx](https://drive.google.com/file/d/1eMGJfdCVlDPBphBTjMcnIh3wdW7Q7WZB/view?usp=sharing)
+
+The model was trained in NuScenes database for 20 epochs.
+
+If you want to train model using the [TIER IV's internal database(~2600 key frames)](https://drive.google.com/file/d/1UaarK88HZu09sf7Ix-bEVl9zGNGFwTVL/view?usp=sharing), please refer to the following repositories:[BEVDet adapted to TIER IV dataset](https://github.com/cyn-liu/BEVDet/tree/train_export)
+
+## References/External links
+
+[1] <https://github.com/HuangJunJie2017/BEVDet/tree/dev2.1>
+
+[2] <https://github.com/LCH1238/BEVDet/tree/export>
+
+[3] <https://github.com/LCH1238/bevdet-tensorrt-cpp/tree/one>

perception/autoware_tensorrt_bevdet/config/bevdet.param.yaml

@@ -0,0 +1,14 @@
+/**:
+    ros__parameters:
+      # weight files
+      onnx_path: "$(var model_path)/$(var model_name).onnx"
+      engine_path: "$(var model_path)/$(var model_name).engine"
+      data_params:
+        N: 6  # camera num
+        H: 900 # image height
+        W: 1600 # image width
+        cams: ["CAM_FRONT_LEFT", "CAM_FRONT", "CAM_FRONT_RIGHT", "CAM_BACK_LEFT", "CAM_BACK", "CAM_BACK_RIGHT"]
+      post_process_params:
+        # post-process params
+        score_threshold: 0.2
+        class_names: ["car", "truck", "construction_vehicle", "bus", "trailer", "barrier", "motorcycle", "bicycle", "pedestrian", "traffic_cone"]

perception/autoware_tensorrt_bevdet/config/bevdet_r50_4dlongterm_depth.yaml

@@ -0,0 +1,63 @@
+bev_range: [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0]
+mean: [123.675, 116.28, 103.53]
+std: [58.395, 57.12, 57.375]
+use_depth: true
+use_adj: true
+adj_num: 8
+transform_size: 6
+cam_params_size: 27
+sampling: nearest # nearest # or bicubic
+data_config:
+  Ncams: 6
+  cams: [CAM_FRONT_LEFT, CAM_FRONT, CAM_FRONT_RIGHT, CAM_BACK_LEFT, CAM_BACK, CAM_BACK_RIGHT]
+  resize_radio: 0.44
+  crop: [140, 0]
+  flip: true
+  input_size: [256, 704]
+  resize: [-0.06, 0.11]
+  resize_test: 0.0
+  rot: [-5.4, 5.4]
+  src_size: [900, 1600]
+grid_config:
+  depth: [1.0, 60.0, 0.5]
+  x: [-51.2, 51.2, 0.8]
+  y: [-51.2, 51.2, 0.8]
+  z: [-5, 3, 8]
+model:
+  bevpool_channels: 80
+  coder:
+    code_size: 9
+    max_num: 500
+    post_center_range: [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0]
+    score_threshold: 0.1
+  common_head:
+    channels: [2, 1, 3, 2, 2]
+    names: [reg, height, dim, rot, vel]
+  down_sample: 16
+  tasks:
+    - class_names:
+        [
+          car,
+          truck,
+          construction_vehicle,
+          bus,
+          trailer,
+          barrier,
+          motorcycle,
+          bicycle,
+          pedestrian,
+          traffic_cone,
+        ]
+      num_class: 10
+test_cfg:
+  max_per_img: 500
+  max_pool_nms: false
+  min_radius: [4, 12, 10, 1, 0.85, 0.175]
+  nms_rescale_factor:
+    - [1.0, 0.7, 0.7, 0.4, 0.55, 1.1, 1.0, 1.0, 1.5, 3.5]
+  nms_thr: [0.2]
+  nms_type: [rotate]
+  post_center_limit_range: [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0]
+  post_max_size: 500
+  pre_max_size: 1000
+  score_threshold: 0.1