This PR adds example of Nvidia Holoscan integration

cpp/camera_stream/CMakeLists.txt

@@ -14,7 +14,8 @@ message(STATUS "Found depthai: ${depthai_DIR}")
 add_executable(main src/main.cpp)
 
 # Link with libraries
-target_link_libraries(main PUBLIC depthai::core)
+target_link_libraries(main PUBLIC depthai::core
+)
 
 # Suppress warnings about C++17 ABI in GCC 10.1+
 target_compile_options(main PRIVATE -Wno-psabi)

integrations/nvidia-holoscan/cpp/.oakappignore

@@ -0,0 +1,10 @@
+# Build artifacts
+build/
+
+# Documentation
+README.md
+
+# VCS
+.git/
+.github/
+.gitlab/

integrations/nvidia-holoscan/cpp/CMakeLists.txt

@@ -0,0 +1,123 @@
+cmake_minimum_required(VERSION 3.20)
+project(nvidia_holoscan_cpp VERSION 1.0 LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+option(HSB_BUILD_RGB_STREAM "Build rgb_stream DepthAI example" ON)
+option(HSB_FETCH_HOLOLINK "Fetch holoscan-sensor-bridge when HSB_SOURCE_DIR is not set" ON)
+option(HSB_FETCH_DLPACK "Download dlpack header if not found locally" ON)
+
+set(HSB_SOURCE_DIR "" CACHE PATH "Path to a holoscan-sensor-bridge checkout")
+set(HSB_REPO_URL "https://github.com/nvidia-holoscan/holoscan-sensor-bridge.git" CACHE STRING "holoscan-sensor-bridge Git URL")
+set(HSB_REPO_TAG "6930609c4ce264ec7e2936dd1f5813323fccb08e" CACHE STRING "holoscan-sensor-bridge commit/tag")
+
+if(HSB_SOURCE_DIR)
+    set(_hsb_root "${HSB_SOURCE_DIR}")
+elseif(HSB_FETCH_HOLOLINK)
+    include(FetchContent)
+    FetchContent_Declare(holoscan_sensor_bridge
+        GIT_REPOSITORY "${HSB_REPO_URL}"
+        GIT_TAG "${HSB_REPO_TAG}"
+        GIT_SHALLOW TRUE
+    )
+    FetchContent_GetProperties(holoscan_sensor_bridge)
+    if(NOT holoscan_sensor_bridge_POPULATED)
+        FetchContent_Populate(holoscan_sensor_bridge)
+    endif()
+    set(_hsb_root "${holoscan_sensor_bridge_SOURCE_DIR}")
+else()
+    message(FATAL_ERROR
+        "HSB_SOURCE_DIR is empty and HSB_FETCH_HOLOLINK=OFF.\n"
+        "Set HSB_SOURCE_DIR to your holoscan-sensor-bridge checkout."
+    )
+endif()
+
+set(_hsb_emulation_dir "${_hsb_root}/src/hololink/emulation")
+if(NOT EXISTS "${_hsb_emulation_dir}/hsb_emulator.hpp")
+    message(FATAL_ERROR
+        "HSB emulation sources not found. Expected:\n"
+        "  ${_hsb_emulation_dir}/hsb_emulator.hpp"
+    )
+endif()
+
+set(HOLOLINK_EMULATION_SOURCES
+    "${_hsb_emulation_dir}/base_transmitter.cpp"
+    "${_hsb_emulation_dir}/data_plane.cpp"
+    "${_hsb_emulation_dir}/hsb_config.cpp"
+    "${_hsb_emulation_dir}/hsb_emulator.cpp"
+    "${_hsb_emulation_dir}/mem_register.cpp"
+    "${_hsb_emulation_dir}/net.cpp"
+    "${_hsb_emulation_dir}/utils.cpp"
+)
+
+find_path(DLPACK_INCLUDE_DIR dlpack/dlpack.h
+    HINTS
+        "${_hsb_root}/build/_deps/dlpack/include"
+)
+if(NOT EXISTS "${DLPACK_INCLUDE_DIR}/dlpack/dlpack.h")
+    if(HSB_FETCH_DLPACK)
+        set(_dlpack_dir "${CMAKE_BINARY_DIR}/_deps/dlpack/include/dlpack")
+        file(MAKE_DIRECTORY "${_dlpack_dir}")
+        file(DOWNLOAD
+            "https://raw.githubusercontent.com/dmlc/dlpack/v1.0/include/dlpack/dlpack.h"
+            "${_dlpack_dir}/dlpack.h"
+            STATUS _dlpack_status
+            SHOW_PROGRESS
+        )
+        list(GET _dlpack_status 0 _dlpack_status_code)
+        if(_dlpack_status_code EQUAL 0)
+            set(DLPACK_INCLUDE_DIR "${CMAKE_BINARY_DIR}/_deps/dlpack/include")
+        endif()
+    endif()
+endif()
+if(NOT EXISTS "${DLPACK_INCLUDE_DIR}/dlpack/dlpack.h")
+    message(FATAL_ERROR
+        "dlpack/dlpack.h not found.\n"
+        "Install dlpack headers, set DLPACK_INCLUDE_DIR, or enable HSB_FETCH_DLPACK."
+    )
+endif()
+
+add_library(hololink_emulation STATIC ${HOLOLINK_EMULATION_SOURCES})
+target_include_directories(hololink_emulation
+    PUBLIC
+        "${_hsb_root}/src"
+        "${_hsb_emulation_dir}"
+        "${DLPACK_INCLUDE_DIR}"
+)
+target_link_libraries(hololink_emulation PUBLIC pthread)
+
+if(HSB_BUILD_RGB_STREAM)
+    find_package(ZLIB REQUIRED)
+    find_package(CUDAToolkit REQUIRED)
+    find_package(depthai REQUIRED)
+    find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
+
+    add_library(hololink_emulation_roce STATIC
+        "${_hsb_emulation_dir}/linux_data_plane.cpp"
+        "${_hsb_emulation_dir}/linux_transmitter.cpp"
+    )
+    target_include_directories(hololink_emulation_roce
+        PUBLIC
+            "${_hsb_root}/src"
+            "${_hsb_emulation_dir}"
+            "${DLPACK_INCLUDE_DIR}"
+    )
+    target_link_libraries(hololink_emulation_roce
+        PUBLIC
+            hololink_emulation
+            ZLIB::ZLIB
+            CUDA::cudart
+    )
+
+    add_executable(rgb_stream rgb_stream.cpp)
+    target_link_libraries(rgb_stream
+        PRIVATE
+            depthai::core
+            hololink_emulation
+            hololink_emulation_roce
+            ${OpenCV_LIBS}
+    )
+    target_compile_options(rgb_stream PRIVATE -Wno-psabi)
+endif()

integrations/nvidia-holoscan/cpp/README.md

@@ -0,0 +1,50 @@
+
+First build the example with (adjust for yourself):
+```
+cmake -S . -B build \
+  -DHSB_SOURCE_DIR=/home/aljaz/work-luxonis/holoscan-sensor-bridge \
+  -DHSB_BUILD_RGB_STREAM=ON \
+  -Ddepthai_DIR=/home/aljaz/work-luxonis/depthai-core/build \
+  -DXLink_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/xlink-build \
+  -Dlibnop_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/libnop-build/lib/cmake/libnop \
+  -Dnlohmann_json_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/nlohmann_json-build \
+  -Dxtensor_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/xtensor-build \
+  -Dxtl_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/xtl-build
+
+cmake --build build -j
+./build/rgb_stream
+
+```
+
+
+Tested on Nvidia jetson orin nano, with JetPack 6.2.1 and cuda 12.6. To install the containers follow the guide:
+1) (host) j[etson setup](https://docs.nvidia.com/holoscan/sensor-bridge/2.0.0/setup.html#sd-tab-item-1): one change is here `sudo nmcli con add con-name hololink-eth0 ifname eth0 type ethernet ip4 192.168.0.101/24` where you set to the IP you have (might have to add a default path aswell). Other change is in `docker/Dockerfile` where you change the version to: `    echo "deb https://repo.download.nvidia.com/jetson/common r36.4 main" > /etc/apt/sources.list.d/nvidia-l4t-apt-source.list && \
+    echo "deb https://repo.download.nvidia.com/jetson/t234 r36.4 main" >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list && \`
+2) then the [demo containers](https://docs.nvidia.com/holoscan/sensor-bridge/2.0.0/build.html): run with `--igpu`. 
+
+Copy the python script from `../jetson/linux_rgb_body_pose.py` to your jetson into the examples. The only difference with the default linux_body_pose_estimation.py example is input is 640 x 640 and the data input is RGB888i instead of IMX sensor data in Bayer format.
+
+to run `examples/linux_rgb_body_pose.py`:
+- get an onnx of the yolov8l model
+- adjust path name in the script
+- change the `examples/linux_rgb_body_pose.yaml` line   `is_engine_path` to `false`
+
+Once setup run the container:
+```
+xhost +
+sh docker/demo.sh
+```
+
+and run the demo with:
+- first run (with sudo for socket access) on your computer / camera: `sudo ./build/rgb_stream`
+- then: `python examples/linux_rgb_body_pose.py --hololink <ip of sender camera>' in the container on jetson, it should auto pickup the UDP packets.
+
+
+
+Some tests:
+- running 640 x 640 uncompressed stream
+- YOLO v8l pose model (converted to tensorRT) and  **not** quantized
+- Nvidia Jetson orin Nano
+
+
+Latency hovers around 700ms and the stream has low FPS (5 - 10 FPS). We attribute this to the Orin Nano device being underpowered and the demo example being unoptimized and the model being unquantized.

integrations/nvidia-holoscan/cpp/rgb_stream.cpp

@@ -0,0 +1,168 @@
+#include <opencv2/core/hal/interface.h>
+
+#include <chrono>
+#include <cstddef>
+#include <cstdio>
+#include <iostream>
+#include <limits>
+#include <map>
+#include <memory>
+#include <opencv2/core.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/opencv.hpp>
+#include <utility>
+
+#include "depthai/depthai.hpp"
+#include "depthai/pipeline/datatype/ImgFrame.hpp"
+#include "hsb_emulator.hpp"
+#include "linux_data_plane.hpp"
+#include "net.hpp"
+
+using namespace hololink::emulation;
+
+int main() {
+  IPAddress emulatorIp = IPAddress_from_string("10.12.101.183");
+
+  std::cout << "IPAddress: " << emulatorIp.if_name << ", "
+            << inet_ntoa(*(in_addr *)&emulatorIp.ip_address) << ", "
+            << inet_ntoa(*(in_addr *)&emulatorIp.subnet_mask) << ", "
+            << inet_ntoa(*(in_addr *)&emulatorIp.broadcast_address) << ", "
+            << std::hex << emulatorIp.mac[0] << ":" << std::hex
+            << emulatorIp.mac[1] << ":" << std::hex << emulatorIp.mac[2] << ":"
+            << std::hex << emulatorIp.mac[3] << ":" << std::hex
+            << emulatorIp.mac[4] << ":" << std::hex << emulatorIp.mac[5]
+            << std::dec << ", port: " << emulatorIp.port
+            << ", flags: " << static_cast<int>(emulatorIp.flags) << std::endl;
+
+  auto device = std::make_shared<dai::Device>();
+
+  // Create pipeline
+  dai::Pipeline pipeline{device};
+
+  // Create and configure camera node
+  auto cameraNode = pipeline.create<dai::node::Camera>();
+  cameraNode->build();
+  auto cameraOut = cameraNode->requestOutput(std::make_pair(640, 640));
+
+  auto qRgb = cameraOut->createOutputQueue();
+
+  auto startTime = std::chrono::steady_clock::now();
+  int counter = 0;
+
+  HSBEmulator hsb;
+  constexpr uint32_t kHifAddressBase = 0x02000300;
+  constexpr uint32_t kHifAddressStep = 0x00010000;
+  constexpr uint32_t kVpAddressBase = 0x00001000;
+  constexpr uint32_t kVpAddressStep = 0x00000040;
+  constexpr uint32_t kDefaultPacketPages = 1;
+  constexpr uint32_t kDefaultQp = 1;
+  constexpr uint32_t kDefaultRkey = 1;
+
+  uint8_t data_plane_id = 0;
+  uint8_t sensor_id = 0;
+  LinuxDataPlane linux_data_plane(hsb, emulatorIp, data_plane_id, sensor_id);
+  const auto hifAddress = kHifAddressBase + kHifAddressStep * data_plane_id;
+  const auto sif0Index =
+      static_cast<uint8_t>(sensor_id * HSB_EMULATOR_CONFIG.sifs_per_sensor);
+  const auto vpAddress = kVpAddressBase + kVpAddressStep * sif0Index;
+
+  // Program default transport metadata. Destination IP/port is typically set by
+  // host-side configuration after enumeration; these defaults ensure payload
+  // metadata is valid.
+  hsb.write(hifAddress + hololink::DP_PACKET_SIZE, kDefaultPacketPages);
+  hsb.write(hifAddress + hololink::DP_PACKET_UDP_PORT,
+            hololink::DATA_SOURCE_UDP_PORT);
+  hsb.write(vpAddress + hololink::DP_QP, kDefaultQp);
+  hsb.write(vpAddress + hololink::DP_RKEY, kDefaultRkey);
+  hsb.write(vpAddress + hololink::DP_BUFFER_MASK, 0x1u);
+  hsb.write(vpAddress + hololink::DP_ADDRESS_0, 0u);
+
+  pipeline.start();
+  hsb.start();
+
+  while (pipeline.isRunning()) {
+    auto inRgb = qRgb->get<dai::ImgFrame>();
+    if (inRgb == nullptr) {
+      std::cerr << "Invalid frame. Skipping." << std::endl;
+      continue;
+    }
+
+    cv::Mat cvFrame = inRgb->getCvFrame();
+    cv::imshow("RGB Frame", cvFrame);
+    auto key = cv::waitKey(1);
+    if (key == 'q') {
+      break;
+    }
+    if (cvFrame.empty()) {
+      std::cerr << "Empty frame payload. Skipping frame." << std::endl;
+      continue;
+    }
+
+    // cv::Mat bayerFrame(cvFrame.rows, cvFrame.cols, CV_8UC1);
+    // for(int y = 0; y < cvFrame.rows; ++y) {
+    //     const cv::Vec3b* srcRow = cvFrame.ptr<cv::Vec3b>(y);
+    //     uint8_t* dstRow = bayerFrame.ptr<uint8_t>(y);
+    //     const bool evenRow = (y & 1) == 0;
+    //     for(int x = 0; x < cvFrame.cols; ++x) {
+    //         const cv::Vec3b& p = srcRow[x];  // BGR
+    //         const bool evenCol = (x & 1) == 0;
+    //         // BGGR Bayer layout:
+    //         // B G
+    //         // G R
+    //         if(evenRow) {
+    //             dstRow[x] = evenCol ? p[0] : p[1];
+    //         } else {
+    //             dstRow[x] = evenCol ? p[1] : p[2];
+    //         }
+    //     }
+    // }
+
+    cv::Mat rgbFrame;
+    cv::cvtColor(cvFrame, rgbFrame, cv::COLOR_BGR2RGB);
+
+    // cv::Mat bayerFrame = cvFrame.clone();
+    // if(!bayerFrame.isContinuous()) {
+    //     bayerFrame = bayerFrame.clone();
+    // }
+
+    const size_t bayerSize = rgbFrame.total() * rgbFrame.elemSize();
+    if (bayerSize == 0) {
+      std::cerr << "Converted Bayer frame is empty. Skipping frame."
+                << std::endl;
+      continue;
+    }
+
+    if (bayerSize > std::numeric_limits<uint32_t>::max()) {
+      std::cerr << "Frame payload too large for DP_BUFFER_LENGTH register. "
+                   "Skipping frame."
+                << std::endl;
+      continue;
+    }
+
+    // Update frame length metadata expected by LinuxDataPlane.
+    hsb.write(vpAddress + hololink::DP_BUFFER_LENGTH,
+              static_cast<uint32_t>(bayerSize));
+
+    int64_t shape[1] = {static_cast<int64_t>(bayerSize)};
+    DLTensor tensor{};
+    tensor.data = rgbFrame.data;
+    tensor.device = {kDLCPU, 0};
+    tensor.ndim = 1;
+    tensor.dtype = {1, 8, 1}; // uint8
+    tensor.shape = shape;
+    tensor.strides = nullptr;
+    tensor.byte_offset = 0;
+
+    const int64_t sentBytes = linux_data_plane.send(tensor);
+    if (sentBytes <= 0) {
+      std::cerr << "Frame not sent yet (waiting for destination configuration)."
+                << std::endl;
+    } else {
+      std::cout << "Sent frame of size " << sentBytes << " bytes." << std::endl;
+    }
+  }
+
+  hsb.stop();
+
+  return 0;
+}