diff --git a/cpp/camera_stream/CMakeLists.txt b/cpp/camera_stream/CMakeLists.txt index f91a52363..105fee407 100644 --- a/cpp/camera_stream/CMakeLists.txt +++ b/cpp/camera_stream/CMakeLists.txt @@ -14,7 +14,8 @@ message(STATUS "Found depthai: ${depthai_DIR}") add_executable(main src/main.cpp) # Link with libraries -target_link_libraries(main PUBLIC depthai::core) +target_link_libraries(main PUBLIC depthai::core +) # Suppress warnings about C++17 ABI in GCC 10.1+ target_compile_options(main PRIVATE -Wno-psabi) diff --git a/integrations/nvidia-holoscan/cpp/.oakappignore b/integrations/nvidia-holoscan/cpp/.oakappignore new file mode 100644 index 000000000..34e630e37 --- /dev/null +++ b/integrations/nvidia-holoscan/cpp/.oakappignore @@ -0,0 +1,10 @@ +# Build artifacts +build/ + +# Documentation +README.md + +# VCS +.git/ +.github/ +.gitlab/ diff --git a/integrations/nvidia-holoscan/cpp/CMakeLists.txt b/integrations/nvidia-holoscan/cpp/CMakeLists.txt new file mode 100644 index 000000000..f5a61f94e --- /dev/null +++ b/integrations/nvidia-holoscan/cpp/CMakeLists.txt @@ -0,0 +1,123 @@ +cmake_minimum_required(VERSION 3.20) +project(nvidia_holoscan_cpp VERSION 1.0 LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +option(HSB_BUILD_RGB_STREAM "Build rgb_stream DepthAI example" ON) +option(HSB_FETCH_HOLOLINK "Fetch holoscan-sensor-bridge when HSB_SOURCE_DIR is not set" ON) +option(HSB_FETCH_DLPACK "Download dlpack header if not found locally" ON) + +set(HSB_SOURCE_DIR "" CACHE PATH "Path to a holoscan-sensor-bridge checkout") +set(HSB_REPO_URL "https://github.com/nvidia-holoscan/holoscan-sensor-bridge.git" CACHE STRING "holoscan-sensor-bridge Git URL") +set(HSB_REPO_TAG "6930609c4ce264ec7e2936dd1f5813323fccb08e" CACHE STRING "holoscan-sensor-bridge commit/tag") + +if(HSB_SOURCE_DIR) + set(_hsb_root "${HSB_SOURCE_DIR}") +elseif(HSB_FETCH_HOLOLINK) + include(FetchContent) + FetchContent_Declare(holoscan_sensor_bridge + GIT_REPOSITORY "${HSB_REPO_URL}" + GIT_TAG "${HSB_REPO_TAG}" + GIT_SHALLOW TRUE + ) + FetchContent_GetProperties(holoscan_sensor_bridge) + if(NOT holoscan_sensor_bridge_POPULATED) + FetchContent_Populate(holoscan_sensor_bridge) + endif() + set(_hsb_root "${holoscan_sensor_bridge_SOURCE_DIR}") +else() + message(FATAL_ERROR + "HSB_SOURCE_DIR is empty and HSB_FETCH_HOLOLINK=OFF.\n" + "Set HSB_SOURCE_DIR to your holoscan-sensor-bridge checkout." + ) +endif() + +set(_hsb_emulation_dir "${_hsb_root}/src/hololink/emulation") +if(NOT EXISTS "${_hsb_emulation_dir}/hsb_emulator.hpp") + message(FATAL_ERROR + "HSB emulation sources not found. Expected:\n" + " ${_hsb_emulation_dir}/hsb_emulator.hpp" + ) +endif() + +set(HOLOLINK_EMULATION_SOURCES + "${_hsb_emulation_dir}/base_transmitter.cpp" + "${_hsb_emulation_dir}/data_plane.cpp" + "${_hsb_emulation_dir}/hsb_config.cpp" + "${_hsb_emulation_dir}/hsb_emulator.cpp" + "${_hsb_emulation_dir}/mem_register.cpp" + "${_hsb_emulation_dir}/net.cpp" + "${_hsb_emulation_dir}/utils.cpp" +) + +find_path(DLPACK_INCLUDE_DIR dlpack/dlpack.h + HINTS + "${_hsb_root}/build/_deps/dlpack/include" +) +if(NOT EXISTS "${DLPACK_INCLUDE_DIR}/dlpack/dlpack.h") + if(HSB_FETCH_DLPACK) + set(_dlpack_dir "${CMAKE_BINARY_DIR}/_deps/dlpack/include/dlpack") + file(MAKE_DIRECTORY "${_dlpack_dir}") + file(DOWNLOAD + "https://raw.githubusercontent.com/dmlc/dlpack/v1.0/include/dlpack/dlpack.h" + "${_dlpack_dir}/dlpack.h" + STATUS _dlpack_status + SHOW_PROGRESS + ) + list(GET _dlpack_status 0 _dlpack_status_code) + if(_dlpack_status_code EQUAL 0) + set(DLPACK_INCLUDE_DIR "${CMAKE_BINARY_DIR}/_deps/dlpack/include") + endif() + endif() +endif() +if(NOT EXISTS "${DLPACK_INCLUDE_DIR}/dlpack/dlpack.h") + message(FATAL_ERROR + "dlpack/dlpack.h not found.\n" + "Install dlpack headers, set DLPACK_INCLUDE_DIR, or enable HSB_FETCH_DLPACK." + ) +endif() + +add_library(hololink_emulation STATIC ${HOLOLINK_EMULATION_SOURCES}) +target_include_directories(hololink_emulation + PUBLIC + "${_hsb_root}/src" + "${_hsb_emulation_dir}" + "${DLPACK_INCLUDE_DIR}" +) +target_link_libraries(hololink_emulation PUBLIC pthread) + +if(HSB_BUILD_RGB_STREAM) + find_package(ZLIB REQUIRED) + find_package(CUDAToolkit REQUIRED) + find_package(depthai REQUIRED) + find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc) + + add_library(hololink_emulation_roce STATIC + "${_hsb_emulation_dir}/linux_data_plane.cpp" + "${_hsb_emulation_dir}/linux_transmitter.cpp" + ) + target_include_directories(hololink_emulation_roce + PUBLIC + "${_hsb_root}/src" + "${_hsb_emulation_dir}" + "${DLPACK_INCLUDE_DIR}" + ) + target_link_libraries(hololink_emulation_roce + PUBLIC + hololink_emulation + ZLIB::ZLIB + CUDA::cudart + ) + + add_executable(rgb_stream rgb_stream.cpp) + target_link_libraries(rgb_stream + PRIVATE + depthai::core + hololink_emulation + hololink_emulation_roce + ${OpenCV_LIBS} + ) + target_compile_options(rgb_stream PRIVATE -Wno-psabi) +endif() diff --git a/integrations/nvidia-holoscan/cpp/README.md b/integrations/nvidia-holoscan/cpp/README.md new file mode 100644 index 000000000..3e125318f --- /dev/null +++ b/integrations/nvidia-holoscan/cpp/README.md @@ -0,0 +1,50 @@ + +First build the example with (adjust for yourself): +``` +cmake -S . -B build \ + -DHSB_SOURCE_DIR=/home/aljaz/work-luxonis/holoscan-sensor-bridge \ + -DHSB_BUILD_RGB_STREAM=ON \ + -Ddepthai_DIR=/home/aljaz/work-luxonis/depthai-core/build \ + -DXLink_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/xlink-build \ + -Dlibnop_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/libnop-build/lib/cmake/libnop \ + -Dnlohmann_json_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/nlohmann_json-build \ + -Dxtensor_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/xtensor-build \ + -Dxtl_DIR=/home/aljaz/work-luxonis/depthai-core/build/_deps/xtl-build + +cmake --build build -j +./build/rgb_stream + +``` + + +Tested on Nvidia jetson orin nano, with JetPack 6.2.1 and cuda 12.6. To install the containers follow the guide: +1) (host) j[etson setup](https://docs.nvidia.com/holoscan/sensor-bridge/2.0.0/setup.html#sd-tab-item-1): one change is here `sudo nmcli con add con-name hololink-eth0 ifname eth0 type ethernet ip4 192.168.0.101/24` where you set to the IP you have (might have to add a default path aswell). Other change is in `docker/Dockerfile` where you change the version to: ` echo "deb https://repo.download.nvidia.com/jetson/common r36.4 main" > /etc/apt/sources.list.d/nvidia-l4t-apt-source.list && \ + echo "deb https://repo.download.nvidia.com/jetson/t234 r36.4 main" >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list && \` +2) then the [demo containers](https://docs.nvidia.com/holoscan/sensor-bridge/2.0.0/build.html): run with `--igpu`. + +Copy the python script from `../jetson/linux_rgb_body_pose.py` to your jetson into the examples. The only difference with the default linux_body_pose_estimation.py example is input is 640 x 640 and the data input is RGB888i instead of IMX sensor data in Bayer format. + +to run `examples/linux_rgb_body_pose.py`: +- get an onnx of the yolov8l model +- adjust path name in the script +- change the `examples/linux_rgb_body_pose.yaml` line `is_engine_path` to `false` + +Once setup run the container: +``` +xhost + +sh docker/demo.sh +``` + +and run the demo with: +- first run (with sudo for socket access) on your computer / camera: `sudo ./build/rgb_stream` +- then: `python examples/linux_rgb_body_pose.py --hololink ' in the container on jetson, it should auto pickup the UDP packets. + + + +Some tests: +- running 640 x 640 uncompressed stream +- YOLO v8l pose model (converted to tensorRT) and **not** quantized +- Nvidia Jetson orin Nano + + +Latency hovers around 700ms and the stream has low FPS (5 - 10 FPS). We attribute this to the Orin Nano device being underpowered and the demo example being unoptimized and the model being unquantized. \ No newline at end of file diff --git a/integrations/nvidia-holoscan/cpp/rgb_stream.cpp b/integrations/nvidia-holoscan/cpp/rgb_stream.cpp new file mode 100644 index 000000000..640855a0e --- /dev/null +++ b/integrations/nvidia-holoscan/cpp/rgb_stream.cpp @@ -0,0 +1,168 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "depthai/depthai.hpp" +#include "depthai/pipeline/datatype/ImgFrame.hpp" +#include "hsb_emulator.hpp" +#include "linux_data_plane.hpp" +#include "net.hpp" + +using namespace hololink::emulation; + +int main() { + IPAddress emulatorIp = IPAddress_from_string("10.12.101.183"); + + std::cout << "IPAddress: " << emulatorIp.if_name << ", " + << inet_ntoa(*(in_addr *)&emulatorIp.ip_address) << ", " + << inet_ntoa(*(in_addr *)&emulatorIp.subnet_mask) << ", " + << inet_ntoa(*(in_addr *)&emulatorIp.broadcast_address) << ", " + << std::hex << emulatorIp.mac[0] << ":" << std::hex + << emulatorIp.mac[1] << ":" << std::hex << emulatorIp.mac[2] << ":" + << std::hex << emulatorIp.mac[3] << ":" << std::hex + << emulatorIp.mac[4] << ":" << std::hex << emulatorIp.mac[5] + << std::dec << ", port: " << emulatorIp.port + << ", flags: " << static_cast(emulatorIp.flags) << std::endl; + + auto device = std::make_shared(); + + // Create pipeline + dai::Pipeline pipeline{device}; + + // Create and configure camera node + auto cameraNode = pipeline.create(); + cameraNode->build(); + auto cameraOut = cameraNode->requestOutput(std::make_pair(640, 640)); + + auto qRgb = cameraOut->createOutputQueue(); + + auto startTime = std::chrono::steady_clock::now(); + int counter = 0; + + HSBEmulator hsb; + constexpr uint32_t kHifAddressBase = 0x02000300; + constexpr uint32_t kHifAddressStep = 0x00010000; + constexpr uint32_t kVpAddressBase = 0x00001000; + constexpr uint32_t kVpAddressStep = 0x00000040; + constexpr uint32_t kDefaultPacketPages = 1; + constexpr uint32_t kDefaultQp = 1; + constexpr uint32_t kDefaultRkey = 1; + + uint8_t data_plane_id = 0; + uint8_t sensor_id = 0; + LinuxDataPlane linux_data_plane(hsb, emulatorIp, data_plane_id, sensor_id); + const auto hifAddress = kHifAddressBase + kHifAddressStep * data_plane_id; + const auto sif0Index = + static_cast(sensor_id * HSB_EMULATOR_CONFIG.sifs_per_sensor); + const auto vpAddress = kVpAddressBase + kVpAddressStep * sif0Index; + + // Program default transport metadata. Destination IP/port is typically set by + // host-side configuration after enumeration; these defaults ensure payload + // metadata is valid. + hsb.write(hifAddress + hololink::DP_PACKET_SIZE, kDefaultPacketPages); + hsb.write(hifAddress + hololink::DP_PACKET_UDP_PORT, + hololink::DATA_SOURCE_UDP_PORT); + hsb.write(vpAddress + hololink::DP_QP, kDefaultQp); + hsb.write(vpAddress + hololink::DP_RKEY, kDefaultRkey); + hsb.write(vpAddress + hololink::DP_BUFFER_MASK, 0x1u); + hsb.write(vpAddress + hololink::DP_ADDRESS_0, 0u); + + pipeline.start(); + hsb.start(); + + while (pipeline.isRunning()) { + auto inRgb = qRgb->get(); + if (inRgb == nullptr) { + std::cerr << "Invalid frame. Skipping." << std::endl; + continue; + } + + cv::Mat cvFrame = inRgb->getCvFrame(); + cv::imshow("RGB Frame", cvFrame); + auto key = cv::waitKey(1); + if (key == 'q') { + break; + } + if (cvFrame.empty()) { + std::cerr << "Empty frame payload. Skipping frame." << std::endl; + continue; + } + + // cv::Mat bayerFrame(cvFrame.rows, cvFrame.cols, CV_8UC1); + // for(int y = 0; y < cvFrame.rows; ++y) { + // const cv::Vec3b* srcRow = cvFrame.ptr(y); + // uint8_t* dstRow = bayerFrame.ptr(y); + // const bool evenRow = (y & 1) == 0; + // for(int x = 0; x < cvFrame.cols; ++x) { + // const cv::Vec3b& p = srcRow[x]; // BGR + // const bool evenCol = (x & 1) == 0; + // // BGGR Bayer layout: + // // B G + // // G R + // if(evenRow) { + // dstRow[x] = evenCol ? p[0] : p[1]; + // } else { + // dstRow[x] = evenCol ? p[1] : p[2]; + // } + // } + // } + + cv::Mat rgbFrame; + cv::cvtColor(cvFrame, rgbFrame, cv::COLOR_BGR2RGB); + + // cv::Mat bayerFrame = cvFrame.clone(); + // if(!bayerFrame.isContinuous()) { + // bayerFrame = bayerFrame.clone(); + // } + + const size_t bayerSize = rgbFrame.total() * rgbFrame.elemSize(); + if (bayerSize == 0) { + std::cerr << "Converted Bayer frame is empty. Skipping frame." + << std::endl; + continue; + } + + if (bayerSize > std::numeric_limits::max()) { + std::cerr << "Frame payload too large for DP_BUFFER_LENGTH register. " + "Skipping frame." + << std::endl; + continue; + } + + // Update frame length metadata expected by LinuxDataPlane. + hsb.write(vpAddress + hololink::DP_BUFFER_LENGTH, + static_cast(bayerSize)); + + int64_t shape[1] = {static_cast(bayerSize)}; + DLTensor tensor{}; + tensor.data = rgbFrame.data; + tensor.device = {kDLCPU, 0}; + tensor.ndim = 1; + tensor.dtype = {1, 8, 1}; // uint8 + tensor.shape = shape; + tensor.strides = nullptr; + tensor.byte_offset = 0; + + const int64_t sentBytes = linux_data_plane.send(tensor); + if (sentBytes <= 0) { + std::cerr << "Frame not sent yet (waiting for destination configuration)." + << std::endl; + } else { + std::cout << "Sent frame of size " << sentBytes << " bytes." << std::endl; + } + } + + hsb.stop(); + + return 0; +} diff --git a/integrations/nvidia-holoscan/jetson/linux_rgb_body_pose.py b/integrations/nvidia-holoscan/jetson/linux_rgb_body_pose.py new file mode 100644 index 000000000..5355e10f1 --- /dev/null +++ b/integrations/nvidia-holoscan/jetson/linux_rgb_body_pose.py @@ -0,0 +1,277 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Body-pose pipeline variant that expects Linux receiver payload as packed +# RGB888 interleaved bytes (H * W * 3). + +import argparse +import logging +import os + +import cuda.bindings.driver as cuda +import cupy as cp +import holoscan +from body_pose_estimation import FormatInferenceInputOp, PostprocessorOp + +import hololink as hololink_module + + +class RgbBytesToImageOp(holoscan.core.Operator): + """Convert packed RGB bytes into an HxWx3 tensor.""" + + def __init__(self, *args, width, height, **kwargs): + super().__init__(*args, **kwargs) + self._width = width + self._height = height + self._expected_size = width * height * 3 + + def setup(self, spec): + spec.input("input") + spec.output("output") + + def compute(self, op_input, op_output, context): + in_message = op_input.receive("input") + frame = cp.asarray(in_message.get(""), dtype=cp.uint8) + if frame.size != self._expected_size: + raise RuntimeError( + f"Expected {self._expected_size} bytes for RGB frame, got {frame.size}" + ) + rgb_image = frame.reshape((self._height, self._width, 3)) + op_output.emit({"": rgb_image}, "output") + + +class StreamControlDevice: + """No-op stream control for externally produced Linux receiver frames.""" + + def start(self): + return + + def stop(self): + return + + +class HoloscanApplication(holoscan.core.Application): + def __init__( + self, + headless, + fullscreen, + cuda_context, + cuda_device_ordinal, + hololink_channel, + stream_width, + stream_height, + stream_device, + frame_limit, + engine, + ): + logging.info("__init__") + super().__init__() + self._headless = headless + self._fullscreen = fullscreen + self._cuda_context = cuda_context + self._cuda_device_ordinal = cuda_device_ordinal + self._hololink_channel = hololink_channel + self._stream_width = stream_width + self._stream_height = stream_height + self._stream_device = stream_device + self._frame_limit = frame_limit + self._engine = engine + + def compose(self): + logging.info("compose") + if self._frame_limit: + self._count = holoscan.conditions.CountCondition( + self, + name="count", + count=self._frame_limit, + ) + condition = self._count + else: + self._ok = holoscan.conditions.BooleanCondition( + self, name="ok", enable_tick=True + ) + condition = self._ok + + frame_size = self._stream_width * self._stream_height * 3 + print(f"Frame size: {frame_size}") + print(f"Stream width: {self._stream_width} Stream height: {self._stream_height}") + + receiver_operator = hololink_module.operators.LinuxReceiverOperator( + self, + condition, + name="receiver", + frame_size=frame_size, + frame_context=self._cuda_context, + hololink_channel=self._hololink_channel, + device=self._stream_device, + ) + rgb_unpack = RgbBytesToImageOp( + self, + name="rgb_unpack", + width=self._stream_width, + height=self._stream_height, + ) + + holoviz_args = self.kwargs("holoviz") + holoviz_args.setdefault("width", self._stream_width) + holoviz_args.setdefault("height", self._stream_height) + + visualizer = holoscan.operators.HolovizOp( + self, + name="holoviz", + fullscreen=self._fullscreen, + headless=self._headless, + framebuffer_srgb=True, + **holoviz_args, + ) + + pool = holoscan.resources.UnboundedAllocator(self) + preprocessor_args = self.kwargs("preprocessor") + preprocessor_args["resize_width"] = self._stream_width + preprocessor_args["resize_height"] = self._stream_height + preprocessor = holoscan.operators.FormatConverterOp( + self, + name="preprocessor", + pool=pool, + **preprocessor_args, + ) + format_input = FormatInferenceInputOp( + self, + name="transpose", + pool=pool, + ) + inference = holoscan.operators.InferenceOp( + self, + name="inference", + allocator=pool, + model_path_map={ + "yolo_pose": self._engine, + }, + **self.kwargs("inference"), + ) + postprocessor_args = self.kwargs("postprocessor") + postprocessor_args["image_dim"] = preprocessor_args["resize_width"] + postprocessor = PostprocessorOp( + self, + name="postprocessor", + allocator=pool, + **postprocessor_args, + ) + + self.add_flow(receiver_operator, rgb_unpack, {("output", "input")}) + self.add_flow(rgb_unpack, visualizer, {("output", "receivers")}) + self.add_flow(rgb_unpack, preprocessor, {("output", "")}) + self.add_flow(preprocessor, format_input) + self.add_flow(format_input, inference, {("", "receivers")}) + self.add_flow(inference, postprocessor, {("transmitter", "in")}) + self.add_flow(postprocessor, visualizer, {("out", "receivers")}) + + # Not using metadata + self.enable_metadata(False) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--stream-width", + type=int, + default=640, + help="Input stream width in pixels", + ) + parser.add_argument( + "--stream-height", + type=int, + default=640, + help="Input stream height in pixels", + ) + parser.add_argument("--headless", action="store_true", help="Run in headless mode") + parser.add_argument( + "--fullscreen", action="store_true", help="Run in fullscreen mode" + ) + parser.add_argument( + "--frame-limit", + type=int, + default=None, + help="Exit after receiving this many frames", + ) + default_configuration = os.path.join( + os.path.dirname(__file__), "body_pose_estimation.yaml" + ) + parser.add_argument( + "--configuration", default=default_configuration, help="Configuration file" + ) + parser.add_argument( + "--hololink", + default="192.168.0.2", + help="IP address of Hololink board", + ) + default_engine = os.path.join(os.path.dirname(__file__), "yolov8l-pose.onnx") + parser.add_argument( + "--engine", + default=default_engine, + help="TRT engine model", + ) + parser.add_argument( + "--log-level", + type=int, + default=20, + help="Logging level to display", + ) + args = parser.parse_args() + if args.stream_width <= 0 or args.stream_height <= 0: + raise ValueError("Stream width and height must be positive integers.") + + hololink_module.logging_level(args.log_level) + logging.info("Initializing.") + + (cu_result,) = cuda.cuInit(0) + assert cu_result == cuda.CUresult.CUDA_SUCCESS + cu_device_ordinal = 0 + cu_result, cu_device = cuda.cuDeviceGet(cu_device_ordinal) + assert cu_result == cuda.CUresult.CUDA_SUCCESS + cu_result, cu_context = cuda.cuDevicePrimaryCtxRetain(cu_device) + assert cu_result == cuda.CUresult.CUDA_SUCCESS + + channel_metadata = hololink_module.Enumerator.find_channel(channel_ip=args.hololink) + hololink_channel = hololink_module.DataChannel(channel_metadata) + stream_device = StreamControlDevice() + application = HoloscanApplication( + args.headless, + args.fullscreen, + cu_context, + cu_device_ordinal, + hololink_channel, + args.stream_width, + args.stream_height, + stream_device, + args.frame_limit, + args.engine, + ) + application.config(args.configuration) + + hololink = hololink_channel.hololink() + hololink.start() + try: + hololink.reset() + application.run() + finally: + hololink.stop() + + (cu_result,) = cuda.cuDevicePrimaryCtxRelease(cu_device) + assert cu_result == cuda.CUresult.CUDA_SUCCESS + + +if __name__ == "__main__": + main() diff --git a/integrations/nvidia-holoscan/python/README.md b/integrations/nvidia-holoscan/python/README.md new file mode 100644 index 000000000..06d47bf27 --- /dev/null +++ b/integrations/nvidia-holoscan/python/README.md @@ -0,0 +1,73 @@ +For full support you need to clone and build the HSB emulator from +[holoscan-sensor-bridge](https://github.com/nvidia-holoscan/holoscan-sensor-bridge.git): + +```bash +git clone https://github.com/nvidia-holoscan/holoscan-sensor-bridge.git +cd holoscan-sensor-bridge +cmake -S src/hololink/emulation -B build +cmake --build build -j +``` + +This build creates a Python virtual environment at `build/env` with the compiled +`hololink.emulation` extension (`_emulation*.so`) installed. + +Install this integration's runtime requirements into that environment and run examples +with the same Python interpreter: + +```bash +# activate your env first +# conda activate hololink-test + +cd /home/aljaz/work-luxonis/oak-examples/integrations/nvidia-holoscan/holoscan-sensor-bridge + +# install the lightweight emulation Python package structure +python -m pip install src/hololink/emulation/python/ + +# build native emulation modules +cmake -S src/hololink/emulation -B build +cmake --build build -j + +# resolve site-packages for current python +PKG_DIR="$(python - <<'PY' +import site +print(next(p for p in site.getsitepackages() if p.endswith('site-packages'))) +PY +)" + +# copy built module(s) into installed package +mkdir -p "$PKG_DIR/hololink/emulation/sensors" +cp build/_emulation*.so "$PKG_DIR/hololink/emulation/" + +SENS_SO="$(find build -name '_emulation_sensors*.so' -print -quit)" +if [ -n "$SENS_SO" ]; then + cp "$SENS_SO" "$PKG_DIR/hololink/emulation/sensors/" +fi + +# verify + run +python -c "from hololink import emulation; print('OK:', emulation.HSBEmulator)" +python /home/aljaz/work-luxonis/oak-examples/integrations/nvidia-holoscan/rgb_stream.py + +``` + + +Tested on Nvidia jetson orin nano, with JetPack 6.2.1 and cuda 12.6. To install the containers follow the guide: +1) (host) j[etson setup](https://docs.nvidia.com/holoscan/sensor-bridge/2.0.0/setup.html#sd-tab-item-1): one change is here `sudo nmcli con add con-name hololink-eth0 ifname eth0 type ethernet ip4 192.168.0.101/24` where you set to the IP you have (might have to add a default path aswell). Other change is in `docker/Dockerfile` where you change the version to: ` echo "deb https://repo.download.nvidia.com/jetson/common r36.4 main" > /etc/apt/sources.list.d/nvidia-l4t-apt-source.list && \ + echo "deb https://repo.download.nvidia.com/jetson/t234 r36.4 main" >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list && \` +2) then the [demo containers](https://docs.nvidia.com/holoscan/sensor-bridge/2.0.0/build.html): run with `--igpu`. + +Copy the python script from `../jetson/linux_rgb_body_pose.py` to your jetson into the examples. The only difference with the default linux_body_pose_estimation.py example is input is 640 x 640 and the data input is RGB888i instead of IMX sensor data in Bayer format. + +to run `examples/linux_rgb_body_pose.py`: +- get an onnx of the yolov8l model +- adjust path name in the script +- change the `examples/linux_rgb_body_pose.yaml` line `is_engine_path` to `false` + +Once setup run the container: +``` +xhost + +sh docker/demo.sh +``` + +and run the demo with: +- first run (with sudo for socket access) on your computer / camera: `sudo ./build/rgb_stream` +- then: `python examples/linux_rgb_body_pose.py --hololink ' in the container on jetson, it should auto pickup the UDP packets. \ No newline at end of file diff --git a/integrations/nvidia-holoscan/python/neural_depth.py b/integrations/nvidia-holoscan/python/neural_depth.py new file mode 100644 index 000000000..7055ce434 --- /dev/null +++ b/integrations/nvidia-holoscan/python/neural_depth.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +import argparse + +import cv2 +import depthai as dai +import numpy as np + +try: + from hololink import emulation +except ModuleNotFoundError as exc: + if exc.name and exc.name.startswith("hololink.emulation._"): + raise ModuleNotFoundError( + "hololink emulation native modules are missing. Build holoscan-sensor-bridge " + "with:\n" + " cmake -S src/hololink/emulation -B build\n" + " cmake --build build -j\n" + "Then run this script with build/env/bin/python." + ) from exc + raise +FPS = 10 + +# Hololink register constants used by the emulator transport metadata setup. +# These values match hololink/core headers. +DP_PACKET_SIZE = 0x04 +DP_PACKET_UDP_PORT = 0x08 +DP_QP = 0x00 +DP_RKEY = 0x04 +DP_ADDRESS_0 = 0x08 +DP_BUFFER_LENGTH = 0x18 +DP_BUFFER_MASK = 0x1C +DATA_SOURCE_UDP_PORT = 12288 + + +def main() -> None: + parser = argparse.ArgumentParser(description="DepthAI -> Hololink emulator bridge") + parser.add_argument("--ip", default="10.12.101.183", help="Local source IP configured on this host") + parser.add_argument("--width", type=int, default=1920) + parser.add_argument("--height", type=int, default=1080) + parser.add_argument("--data-plane-id", type=int, default=0) + parser.add_argument("--sensor-id", type=int, default=0) + args = parser.parse_args() + + data_plane_id = int(args.data_plane_id) + sensor_id = int(args.sensor_id) + + emulator_ip = emulation.IPAddress(args.ip) + emulator = emulation.HSBEmulator() + + k_hif_address_base = 0x02000300 + k_hif_address_step = 0x00010000 + k_vp_address_base = 0x00001000 + k_vp_address_step = 0x00000040 + k_default_packet_pages = 1 + k_default_qp = 1 + k_default_rkey = 1 + + hif_address = k_hif_address_base + k_hif_address_step * data_plane_id + sif0_index = int(sensor_id * emulation.HSB_EMULATOR_CONFIG.sifs_per_sensor) + vp_address = k_vp_address_base + k_vp_address_step * sif0_index + + # Program default transport metadata expected by LinuxDataPlane. + emulator.write(hif_address + DP_PACKET_SIZE, k_default_packet_pages) + emulator.write(hif_address + DP_PACKET_UDP_PORT, DATA_SOURCE_UDP_PORT) + emulator.write(vp_address + DP_QP, k_default_qp) + emulator.write(vp_address + DP_RKEY, k_default_rkey) + emulator.write(vp_address + DP_BUFFER_MASK, 0x1) + emulator.write(vp_address + DP_ADDRESS_0, 0) + + linux_data_plane = emulation.LinuxDataPlane( + hsb_emulator=emulator, + source_ip=emulator_ip, + data_plane_id=data_plane_id, + sensor_id=sensor_id, + ) + + device = dai.Device() + with dai.Pipeline(device) as pipeline: + + cameraLeft = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B, sensorFps=FPS) + cameraRight = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C, sensorFps=FPS) + leftOutput = cameraLeft.requestFullResolutionOutput() + rightOutput = cameraRight.requestFullResolutionOutput() + + neuralDepth = pipeline.create(dai.node.NeuralDepth).build(leftOutput, rightOutput, dai.DeviceModelZoo.NEURAL_DEPTH_LARGE) + + depthOutputQ = neuralDepth.depth.createOutputQueue() + colorMap = cv2.applyColorMap(np.arange(256, dtype=np.uint8), cv2.COLORMAP_JET) + + pipeline.start() + emulator.start() + + while pipeline.isRunning(): + in_rgb = depthOutputQ.get() + if in_rgb is None: + continue + assert isinstance(in_rgb, dai.ImgFrame) + + depth_frame = in_rgb.getCvFrame() # should be a RAW16 (eg uint16) single-channel depth frame + if depth_frame is None or depth_frame.size == 0: + continue + + bgr_frame = cv2.applyColorMap(((depth_frame)).astype(np.uint8), colorMap) + rgb_frame = cv2.cvtColor(bgr_frame, cv2.COLOR_BGR2RGB) + + payload = np.ascontiguousarray(rgb_frame).reshape(-1) + + emulator.write(vp_address + DP_BUFFER_LENGTH, int(payload.nbytes)) + sent_bytes = linux_data_plane.send(payload) + + if sent_bytes <= 0: + print("Frame not sent yet (waiting for destination configuration).") + else: + print(f"Sent {sent_bytes} bytes.") + + cv2.imshow("RGB Frame", bgr_frame) + if cv2.waitKey(1) == ord("q"): + break + + +if __name__ == "__main__": + main() diff --git a/integrations/nvidia-holoscan/python/requirements.txt b/integrations/nvidia-holoscan/python/requirements.txt new file mode 100644 index 000000000..550c9f5c7 --- /dev/null +++ b/integrations/nvidia-holoscan/python/requirements.txt @@ -0,0 +1,5 @@ + +# Runtime dependencies for examples/cpp/Emulation/hololink_emulator.py +numpy>=1.24 +opencv-python>=4.8 +depthai>=3.3.0 diff --git a/integrations/nvidia-holoscan/python/rgb_stream.py b/integrations/nvidia-holoscan/python/rgb_stream.py new file mode 100644 index 000000000..9cca7d02a --- /dev/null +++ b/integrations/nvidia-holoscan/python/rgb_stream.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +import argparse + +import cv2 +import depthai as dai +import numpy as np + +try: + from hololink import emulation +except ModuleNotFoundError as exc: + if exc.name and exc.name.startswith("hololink.emulation._"): + raise ModuleNotFoundError( + "hololink emulation native modules are missing. Build holoscan-sensor-bridge " + "with:\n" + " cmake -S src/hololink/emulation -B build\n" + " cmake --build build -j\n" + "Then run this script with build/env/bin/python." + ) from exc + raise + +# Hololink register constants used by the emulator transport metadata setup. +# These values match hololink/core headers. +DP_PACKET_SIZE = 0x04 +DP_PACKET_UDP_PORT = 0x08 +DP_QP = 0x00 +DP_RKEY = 0x04 +DP_ADDRESS_0 = 0x08 +DP_BUFFER_LENGTH = 0x18 +DP_BUFFER_MASK = 0x1C +DATA_SOURCE_UDP_PORT = 12288 + + +def main() -> None: + parser = argparse.ArgumentParser(description="DepthAI -> Hololink emulator bridge") + parser.add_argument("--ip", default="10.12.101.183", help="Local source IP configured on this host") + parser.add_argument("--width", type=int, default=1920) + parser.add_argument("--height", type=int, default=1080) + parser.add_argument("--data-plane-id", type=int, default=0) + parser.add_argument("--sensor-id", type=int, default=0) + args = parser.parse_args() + + data_plane_id = int(args.data_plane_id) + sensor_id = int(args.sensor_id) + + emulator_ip = emulation.IPAddress(args.ip) + emulator = emulation.HSBEmulator() + + k_hif_address_base = 0x02000300 + k_hif_address_step = 0x00010000 + k_vp_address_base = 0x00001000 + k_vp_address_step = 0x00000040 + k_default_packet_pages = 1 + k_default_qp = 1 + k_default_rkey = 1 + + hif_address = k_hif_address_base + k_hif_address_step * data_plane_id + sif0_index = int(sensor_id * emulation.HSB_EMULATOR_CONFIG.sifs_per_sensor) + vp_address = k_vp_address_base + k_vp_address_step * sif0_index + + # Program default transport metadata expected by LinuxDataPlane. + emulator.write(hif_address + DP_PACKET_SIZE, k_default_packet_pages) + emulator.write(hif_address + DP_PACKET_UDP_PORT, DATA_SOURCE_UDP_PORT) + emulator.write(vp_address + DP_QP, k_default_qp) + emulator.write(vp_address + DP_RKEY, k_default_rkey) + emulator.write(vp_address + DP_BUFFER_MASK, 0x1) + emulator.write(vp_address + DP_ADDRESS_0, 0) + + linux_data_plane = emulation.LinuxDataPlane( + hsb_emulator=emulator, + source_ip=emulator_ip, + data_plane_id=data_plane_id, + sensor_id=sensor_id, + ) + + device = dai.Device() + with dai.Pipeline(device) as pipeline: + camera_node = pipeline.create(dai.node.Camera) + camera_node.build() + camera_out = camera_node.requestOutput((args.width, args.height)) + q_rgb = camera_out.createOutputQueue() + + pipeline.start() + emulator.start() + + try: + while pipeline.isRunning(): + in_rgb = q_rgb.get() + if in_rgb is None: + continue + + bgr_frame = in_rgb.getCvFrame() + if bgr_frame is None or bgr_frame.size == 0: + continue + + rgb_frame = cv2.cvtColor(bgr_frame, cv2.COLOR_BGR2RGB) + payload = np.ascontiguousarray(rgb_frame).reshape(-1) + + emulator.write(vp_address + DP_BUFFER_LENGTH, int(payload.nbytes)) + sent_bytes = linux_data_plane.send(payload) + + if sent_bytes <= 0: + print("Frame not sent yet (waiting for destination configuration).") + else: + print(f"Sent {sent_bytes} bytes.") + + cv2.imshow("RGB Frame", bgr_frame) + if cv2.waitKey(1) == ord("q"): + break + finally: + emulator.stop() + cv2.destroyAllWindows() + + +if __name__ == "__main__": + main()