From ccfd66479a25f577971b2f0ae195754c10064f0b Mon Sep 17 00:00:00 2001 From: wei <1774366232@qq.com> Date: Mon, 2 Feb 2026 13:35:13 +0800 Subject: [PATCH] feat(ppyoloe_plus): Implement ppyoloe_plus detection --- CMakeLists.txt | 1 + element/algorithm/ppyoloe_plus/CMakeLists.txt | 87 ++++++ element/algorithm/ppyoloe_plus/README.md | 62 ++++ .../ppyoloe_plus/include/ppyoloe_plus.h | 108 +++++++ .../include/ppyoloe_plus_context.h | 70 +++++ .../include/ppyoloe_plus_inference.h | 40 +++ .../include/ppyoloe_plus_post_process.h | 47 +++ .../include/ppyoloe_plus_pre_process.h | 38 +++ .../ppyoloe_plus/src/ppyoloe_plus.cc | 277 ++++++++++++++++++ .../src/ppyoloe_plus_inference.cc | 50 ++++ .../src/ppyoloe_plus_post_process.cc | 165 +++++++++++ .../src/ppyoloe_plus_pre_process.cc | 208 +++++++++++++ 12 files changed, 1153 insertions(+) create mode 100644 element/algorithm/ppyoloe_plus/CMakeLists.txt create mode 100644 element/algorithm/ppyoloe_plus/README.md create mode 100644 element/algorithm/ppyoloe_plus/include/ppyoloe_plus.h create mode 100644 element/algorithm/ppyoloe_plus/include/ppyoloe_plus_context.h create mode 100644 element/algorithm/ppyoloe_plus/include/ppyoloe_plus_inference.h create mode 100644 element/algorithm/ppyoloe_plus/include/ppyoloe_plus_post_process.h create mode 100644 element/algorithm/ppyoloe_plus/include/ppyoloe_plus_pre_process.h create mode 100644 element/algorithm/ppyoloe_plus/src/ppyoloe_plus.cc create mode 100644 element/algorithm/ppyoloe_plus/src/ppyoloe_plus_inference.cc create mode 100644 element/algorithm/ppyoloe_plus/src/ppyoloe_plus_post_process.cc create mode 100644 element/algorithm/ppyoloe_plus/src/ppyoloe_plus_pre_process.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 0642aee..61c6497 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,7 @@ checkAndAddElement(element/algorithm/lprnet) checkAndAddElement(element/algorithm/ppocr) checkAndAddElement(element/algorithm/yolov8) checkAndAddElement(element/algorithm/lightstereo) +checkAndAddElement(element/algorithm/ppyoloe_plus) checkAndAddElement(element/multimedia/decode) checkAndAddElement(element/multimedia/osd) diff --git a/element/algorithm/ppyoloe_plus/CMakeLists.txt b/element/algorithm/ppyoloe_plus/CMakeLists.txt new file mode 100644 index 0000000..ffc30f2 --- /dev/null +++ b/element/algorithm/ppyoloe_plus/CMakeLists.txt @@ -0,0 +1,87 @@ +cmake_minimum_required(VERSION 3.10) +project(algorithmAPI) +set(CMAKE_CXX_STANDARD 17) + +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fprofile-arcs -g") + +if (NOT DEFINED TARGET_ARCH) + set(TARGET_ARCH pcie) +endif() + +if (${TARGET_ARCH} STREQUAL "pcie") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -pthread") + + set(FFMPEG_DIR /opt/sophon/sophon-ffmpeg-latest/lib/cmake) + find_package(FFMPEG REQUIRED) + include_directories(${FFMPEG_INCLUDE_DIRS}) + link_directories(${FFMPEG_LIB_DIRS}) + + set(OpenCV_DIR /opt/sophon/sophon-opencv-latest/lib/cmake/opencv4) + find_package(OpenCV REQUIRED) + include_directories(${OpenCV_INCLUDE_DIRS}) + link_directories(${OpenCV_LIB_DIRS}) + + set(LIBSOPHON_DIR /opt/sophon/libsophon-current/data/libsophon-config.cmake) + find_package(LIBSOPHON REQUIRED) + include_directories(${LIBSOPHON_INCLUDE_DIRS}) + link_directories(${LIBSOPHON_LIB_DIRS}) + + set(BM_LIBS bmlib bmrt bmcv yuv) + find_library(BMJPU bmjpuapi) + if(BMJPU) + set(JPU_LIBS bmjpuapi bmjpulite) + endif() + + include_directories(../) + include_directories(../../../framework) + include_directories(../../../framework/include) + + include_directories(../../../3rdparty/spdlog/include) + include_directories(../../../3rdparty/nlohmann-json/include) + include_directories(../../../3rdparty/httplib) + + include_directories(include) + add_library(ppyoloe_plus SHARED + src/ppyoloe_plus_pre_process.cc + src/ppyoloe_plus_post_process.cc + src/ppyoloe_plus_inference.cc + src/ppyoloe_plus.cc + ) + + target_link_libraries(ppyoloe_plus ${FFMPEG_LIBS} ${OpenCV_LIBS} ${BM_LIBS} ${JPU_LIBS} -lpthread) + +elseif (${TARGET_ARCH} STREQUAL "soc") + add_compile_options(-fPIC) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fprofile-arcs -ftest-coverage -g -rdynamic") + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-arcs -ftest-coverage -rdynamic") + set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc) + set(CMAKE_ASM_COMPILER aarch64-linux-gnu-gcc) + set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++) + + include_directories("${SOPHON_SDK_SOC}/include/") + include_directories("${SOPHON_SDK_SOC}/include/opencv4") + link_directories("${SOPHON_SDK_SOC}/lib/") + set(BM_LIBS bmlib bmrt bmcv yuv) + find_library(BMJPU bmjpuapi) + if(BMJPU) + set(JPU_LIBS bmjpuapi bmjpulite) + endif() + + include_directories(../) + include_directories(../../../framework) + include_directories(../../../framework/include) + + include_directories(../../../3rdparty/spdlog/include) + include_directories(../../../3rdparty/nlohmann-json/include) + include_directories(../../../3rdparty/httplib) + + include_directories(include) + add_library(ppyoloe_plus SHARED + src/ppyoloe_plus_pre_process.cc + src/ppyoloe_plus_post_process.cc + src/ppyoloe_plus_inference.cc + src/ppyoloe_plus.cc + ) + target_link_libraries(ppyoloe_plus ${FFMPEG_LIBS} ${OpenCV_LIBS} ${BM_LIBS} ${JPU_LIBS} -fprofile-arcs -lgcov -lpthread) +endif() diff --git a/element/algorithm/ppyoloe_plus/README.md b/element/algorithm/ppyoloe_plus/README.md new file mode 100644 index 0000000..de8843f --- /dev/null +++ b/element/algorithm/ppyoloe_plus/README.md @@ -0,0 +1,62 @@ +# sophon-stream ppyoloe_plus element + +[English](README_EN.md) | 简体中文 + +sophon-stream ppyoloe_plus element是sophon-stream框架中的一个插件，是一个简单、快速、强大的检测模型。本项目已提供此插件例程 + +## 1. 特性 +* 支持多路视频流 +* 支持多线程处理 + +## 2. 配置参数 +sophon-stream ppyoloe_plus插件具有一些可配置的参数，可以根据需求进行设置。以下是一些常用的参数： + +```json +{ + "configure": { + "model_path": "../ppyoloe_bytetrack_ppyoloe_plus/data/models/tray_det_1209_1b.bmodel", + "threshold_conf": 0.65, + "threshold_nms": 0.45, + "bgr2rgb": true, + "mean": [ + 0, + 0, + 0 + ], + "std": [ + 1, + 1, + 1 + ], + "stage": [ + "post" + ], + "use_tpu_kernel": false, + "class_names_file": "../ppyoloe_bytetrack_ppyoloe_plus/data/model.name" + }, + "shared_object": "../../build/lib/libppyoloe_plus.so", + "name": "ppyoloe_plus_post", + "side": "sophgo", + "thread_number": 1 +} +``` + +| 参数名 | 类型 | 默认值 | 说明 | +|:-------------:| :-------: | :------------------:| :------------------------:| +| model_path | 字符串 | "../ppyoloe_bytetrack_ppyoloe_plus/data/models/tray_det_1209_1b.bmodel" | yolov8模型路径 | +| threshold_conf | 浮点数或map | 0.65 | 目标检测物体置信度阈值，设置为浮点数时，所有类别共用同一个阈值；设置为map时，不同类别可以使用不同阈值，此时还需要正确设置class_names_file | +| threshold_nms | 浮点数 | 0.45 | 目标检测NMS IOU阈值 | +| bgr2rgb | bool | true | 解码器解出来的图像默认是bgr格式，是否需要将图像转换成rgb格式 | +| mean | 浮点数组 | 无 | 图像前处理均值，长度为3；计算方式为: y=(x-mean)/std；若bgr2rgb=true，数组中数组顺序需为r、g、b，否则需为b、g、r | +| std | 浮点数组 | 无 | 图像前处理方差，长度为3；计算方式同上；若bgr2rgb=true数组中数组顺序需为r、g、b，否则需为b、g、r | +| stage | 列表 | ["pre"] | 标志前处理、推理、后处理三个阶段 | +| use_tpu_kernel | 布尔值 | true | 是否启用tpu_kernel后处理 | +| class_names_file | 字符串 | 无 | threshold_conf为浮点数时不生效，可以不设置；当threshold_conf为map时启用，class name文件的路径 | +| shared_object | 字符串 | "../../build/lib/libppyoloe_plus.so" | libyolov8 动态库路径 | +| name | 字符串 | "ppyoloe_plus_post" | element 名称 | +| side | 字符串 | "sophgo"| 设备类型 | +| thread_number | 整数 | 1 | 启动线程数 | + +> **注意**： +1. stage参数，需要设置为"pre"，"infer"，"post" 其中之一或相邻项的组合，并且按前处理-推理-后处理的顺序连接element。将三个阶段分配在三个element上的目的是充分利用各项资源，提高检测效率。 + diff --git a/element/algorithm/ppyoloe_plus/include/ppyoloe_plus.h b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus.h new file mode 100644 index 0000000..1bc38a6 --- /dev/null +++ b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus.h @@ -0,0 +1,108 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (C) 2022 Sophgo Technologies Inc. All rights reserved. +// +// SOPHON-STREAM is licensed under the 2-Clause BSD License except for the +// third-party components. +// +//===----------------------------------------------------------------------===// + +#ifndef SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_H_ +#define SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_H_ + +#include "element_factory.h" +#include "group.h" +#include "ppyoloe_plus_context.h" +#include "ppyoloe_plus_inference.h" +#include "ppyoloe_plus_post_process.h" +#include "ppyoloe_plus_pre_process.h" + +namespace sophon_stream { +namespace element { +namespace ppyoloe_plus { + + class Ppyoloe_plus : public ::sophon_stream::framework::Element { + public: + Ppyoloe_plus(); + ~Ppyoloe_plus() override; + + const static std::string elementName; + + /** + * @brief + * 解析configure，初始化派生element的特有属性；调用initContext初始化算法相关参数 + * @param json json格式的配置文件 + * @return common::ErrorCode + * 成功返回common::ErrorCode::SUCCESS，失败返回common::ErrorCode::PARSE_CONFIGURE_FAIL + */ + common::ErrorCode initInternal(const std::string& json) override; + + /** + * @brief + * element的功能在这里实现。例如，算法模块需要实现组batch、调用算法、发送数据等功能 + * @param dataPipeId pop数据时对应的dataPipeId + * @return common::ErrorCode 成功返回common::ErrorCode::SUCCESS + */ + common::ErrorCode doWork(int dataPipeId) override; + + void setContext(std::shared_ptr<::sophon_stream::element::Context> context); + void setPreprocess( + std::shared_ptr<::sophon_stream::element::PreProcess> pre); + void setInference( + std::shared_ptr<::sophon_stream::element::Inference> infer); + void setPostprocess( + std::shared_ptr<::sophon_stream::element::PostProcess> post); + void setStage(bool pre, bool infer, bool post); + void initProfiler(std::string name, int interval); + std::shared_ptr<::sophon_stream::element::Context> getContext() { + return mContext; + } + std::shared_ptr<::sophon_stream::element::PreProcess> getPreProcess() { + return mPreProcess; + } + std::shared_ptr<::sophon_stream::element::Inference> getInference() { + return mInference; + } + std::shared_ptr<::sophon_stream::element::PostProcess> getPostProcess() { + return mPostProcess; + } + + /** + * @brief 从json文件读取的配置项 + */ + static constexpr const char* CONFIG_INTERNAL_STAGE_NAME_FIELD = "stage"; + static constexpr const char* CONFIG_INTERNAL_MODEL_PATH_FIELD = + "model_path"; + static constexpr const char* CONFIG_INTERNAL_THRESHOLD_CONF_FIELD = + "threshold_conf"; + static constexpr const char* CONFIG_INTERNAL_THRESHOLD_NMS_FIELD = + "threshold_nms"; + static constexpr const char* CONFIG_INTERNAL_THRESHOLD_BGR2RGB_FIELD = + "bgr2rgb"; + static constexpr const char* CONFIG_INTERNAL_THRESHOLD_MEAN_FIELD = "mean"; + static constexpr const char* CONFIG_INTERNAL_THRESHOLD_STD_FIELD = "std"; + static constexpr const char* CONFIG_INTERNAL_CLASS_NAMES_FILE_FILED = + "class_names_file"; + + private: + std::shared_ptr mContext; // context对象 + std::shared_ptr mPreProcess; // 预处理对象 + std::shared_ptr mInference; // 推理对象 + std::shared_ptr mPostProcess; // 后处理对象 + + bool use_pre = false; + bool use_infer = false; + bool use_post = false; + + std::string mFpsProfilerName; + ::sophon_stream::common::FpsProfiler mFpsProfiler; + + common::ErrorCode initContext(const std::string& json); + void process(common::ObjectMetadatas& objectMetadatas); + }; + +} // namespace ppyoloe_plus +} // namespace element +} // namespace sophon_stream + +#endif // SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_H_ \ No newline at end of file diff --git a/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_context.h b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_context.h new file mode 100644 index 0000000..fd1418e --- /dev/null +++ b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_context.h @@ -0,0 +1,70 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (C) 2022 Sophgo Technologies Inc. All rights reserved. +// +// SOPHON-STREAM is licensed under the 2-Clause BSD License except for the +// third-party components. +// +//===----------------------------------------------------------------------===// + +#ifndef SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_CONTEXT_H_ +#define SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_CONTEXT_H_ + +#include "algorithmApi/context.h" + +namespace sophon_stream { +namespace element { +namespace ppyoloe_plus { + +#define USE_ASPECT_RATIO 1 + +class Ppyoloe_plusContext : public ::sophon_stream::element::Context { + public: + int deviceId; // 设备ID + + std::shared_ptr bmContext; + std::shared_ptr bmNetwork; + bm_handle_t handle; + + std::vector mean; // 前处理均值， 长度为3，顺序为rgb + std::vector stdd; // 前处理方差， 长度为3，顺序为rgb + bool bgr2rgb; // 是否将bgr图像转成rgb推理 + bmcv_convert_to_attr converto_attr; // 图片做归一化，标准化预处理使用 + + /** + * @brief 最小的置信度阈值。详细说明请参考README + */ + float thresh_conf_min = 0.5; + /** + * @brief 置信度阈值，key：类名，value：阈值 + * 该参数支持对不同的类别设置不同的阈值 + */ + std::unordered_map thresh_conf; + /** + * @brief NMS IOU阈值 + */ + float thresh_nms; + std::vector class_names; + /** + * @brief 决定是否启用类别阈值 + */ + bool class_thresh_valid = false; + + /** + * @brief + * 类别数量，从model中读取。需要和thresh_conf、class_names的长度做校验 + */ + int class_num = 1; + int m_frame_h, m_frame_w; + int net_h, net_w, m_net_channel; + int max_batch; + int input_num; + int output_num; + int min_dim; + +}; +} // namespace ppyoloe_plus +} // namespace element +} // namespace sophon_stream + +#endif // SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_CONTEXT_H_ \ No newline at end of file diff --git a/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_inference.h b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_inference.h new file mode 100644 index 0000000..ae6ff27 --- /dev/null +++ b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_inference.h @@ -0,0 +1,40 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (C) 2022 Sophgo Technologies Inc. All rights reserved. +// +// SOPHON-STREAM is licensed under the 2-Clause BSD License except for the +// third-party components. +// +//===----------------------------------------------------------------------===// + +#ifndef SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_INFERENCE_H_ +#define SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_INFERENCE_H_ + +#include "algorithmApi/inference.h" +#include "ppyoloe_plus_context.h" + +namespace sophon_stream { +namespace element { +namespace ppyoloe_plus { + +class Ppyoloe_plusInference : public ::sophon_stream::element::Inference { + public: + ~Ppyoloe_plusInference() override; + /** + * @brief init device and engine + * @param[in] context: model path,inputs and outputs name... + */ + void init(std::shared_ptr context); + /** + * @brief network predict output + * @param[in] context: inputData and outputData + */ + common::ErrorCode predict(std::shared_ptr context, + common::ObjectMetadatas& objectMetadatas); +}; + +} // namespace ppyoloe_plus +} // namespace element +} // namespace sophon_stream + +#endif // SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_INFERENCE_H_ \ No newline at end of file diff --git a/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_post_process.h b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_post_process.h new file mode 100644 index 0000000..09a423f --- /dev/null +++ b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_post_process.h @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (C) 2022 Sophgo Technologies Inc. All rights reserved. +// +// SOPHON-STREAM is licensed under the 2-Clause BSD License except for the +// third-party components. +// +//===----------------------------------------------------------------------===// + +#ifndef SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_POST_PROCESS_H_ +#define SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_POST_PROCESS_H_ + +#include "algorithmApi/post_process.h" +#include "ppyoloe_plus_context.h" + +namespace sophon_stream { +namespace element { +namespace ppyoloe_plus { + +struct ppYoloePlusBox { + int x, y, width, height; + float score; + int class_id; +}; + +using ppYoloePlusBoxVec = std::vector; + +class Ppyoloe_plusPostProcess : public ::sophon_stream::element::PostProcess { + public: + void init(std::shared_ptr context); + /** + * @brief 对一个batch的数据做后处理 + * @param context context指针 + * @param objectMetadatas 一个batch的数据 + */ + void postProcess(std::shared_ptr context, + common::ObjectMetadatas& objectMetadatas); + private: + int argmax_interval(float *data, int class_num, int box_num); + void NMS(ppYoloePlusBoxVec& dets, float nmsConfidence); +}; + +} // namespace ppyoloe_plus +} // namespace element +} // namespace sophon_stream + +#endif // SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_POST_PROCESS_H_ \ No newline at end of file diff --git a/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_pre_process.h b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_pre_process.h new file mode 100644 index 0000000..28fc0ea --- /dev/null +++ b/element/algorithm/ppyoloe_plus/include/ppyoloe_plus_pre_process.h @@ -0,0 +1,38 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (C) 2022 Sophgo Technologies Inc. All rights reserved. +// +// SOPHON-STREAM is licensed under the 2-Clause BSD License except for the +// third-party components. +// +//===----------------------------------------------------------------------===// + +#ifndef SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_PRE_PROCESS_H_ +#define SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_PRE_PROCESS_H_ + +#include "algorithmApi/pre_process.h" +#include "ppyoloe_plus_context.h" + +namespace sophon_stream { +namespace element { +namespace ppyoloe_plus { + +class Ppyoloe_plusPreProcess : public ::sophon_stream::element::PreProcess { + public: + /** + * @brief 对一个batch的数据做预处理 + * @param context context指针 + * @param objectMetadatas 一个batch的数据 + * @return common::ErrorCode + * common::ErrorCode::SUCCESS，中间过程失败会中断执行 + */ + common::ErrorCode preProcess(std::shared_ptr context, + common::ObjectMetadatas& objectMetadatas); + void init(std::shared_ptr context); +}; + +} // namespace ppyoloe_plus +} // namespace element +} // namespace sophon_stream + +#endif // SOPHON_STREAM_ELEMENT_PPYOLOE_PLUS_PRE_PROCESS_H_ \ No newline at end of file diff --git a/element/algorithm/ppyoloe_plus/src/ppyoloe_plus.cc b/element/algorithm/ppyoloe_plus/src/ppyoloe_plus.cc new file mode 100644 index 0000000..7fc3c40 --- /dev/null +++ b/element/algorithm/ppyoloe_plus/src/ppyoloe_plus.cc @@ -0,0 +1,277 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (C) 2022 Sophgo Technologies Inc. All rights reserved. +// +// SOPHON-STREAM is licensed under the 2-Clause BSD License except for the +// third-party components. +// +//===----------------------------------------------------------------------===// + +#include "ppyoloe_plus.h" + +using namespace std::chrono_literals; + +namespace sophon_stream { +namespace element { +namespace ppyoloe_plus { + + Ppyoloe_plus::Ppyoloe_plus() {} + + Ppyoloe_plus::~Ppyoloe_plus() {} + + const std::string Ppyoloe_plus::elementName = "ppyoloe_plus"; + + common::ErrorCode Ppyoloe_plus::initContext(const std::string& json) { + common::ErrorCode errorCode = common::ErrorCode::SUCCESS; + do { + auto configure = nlohmann::json::parse(json, nullptr, false); + if (!configure.is_object()) { + errorCode = common::ErrorCode::PARSE_CONFIGURE_FAIL; + break; + } + + // class_name + auto class_names_file = configure.find(CONFIG_INTERNAL_CLASS_NAMES_FILE_FILED)->get(); + std::ifstream istream; + istream.open(class_names_file); + assert(istream.is_open()); + std::string line; + while (std::getline(istream, line)) { + line = line.substr(0, line.length()); + mContext->class_names.push_back(line); + } + istream.close(); + + // 置信度阈值 + auto threshConfIt = configure.find(CONFIG_INTERNAL_THRESHOLD_CONF_FIELD); + if (threshConfIt->is_number_float()) { + mContext->thresh_conf_min = threshConfIt->get(); + } + // MNS阈值 + auto threshNmsIt = configure.find(CONFIG_INTERNAL_THRESHOLD_NMS_FIELD); + mContext->thresh_nms = threshNmsIt->get(); + // bgr转rgb + mContext->bgr2rgb = true; + auto bgr2rgbIt = configure.find(CONFIG_INTERNAL_THRESHOLD_BGR2RGB_FIELD); + mContext->bgr2rgb = bgr2rgbIt->get(); + // mean + auto meanIt = configure.find(CONFIG_INTERNAL_THRESHOLD_MEAN_FIELD); + mContext->mean = meanIt->get>(); + assert(mContext->mean.size() == 3); + // strand + auto stdIt = configure.find(CONFIG_INTERNAL_THRESHOLD_STD_FIELD); + mContext->stdd = stdIt->get>(); + assert(mContext->stdd.size() == 3); + + // 1. get network + auto modelPathIt = configure.find(CONFIG_INTERNAL_MODEL_PATH_FIELD); + BMNNHandlePtr handle = std::make_shared(mContext->deviceId); + mContext->bmContext = std::make_shared(handle, modelPathIt->get().c_str()); + mContext->bmNetwork = mContext->bmContext->network(0); + mContext->handle = handle->handle(); + + // 2. get input（参考sophon-demo中ppyoloe C++的实现） + mContext->max_batch = mContext->bmNetwork->maxBatch(); + auto input_img = mContext->bmNetwork->inputTensor(0); // 图片数据 + std::shared_ptr input_ratio = mContext->bmNetwork->inputTensor(1); // 缩放比例 + mContext->input_num = mContext->bmNetwork->m_netinfo->input_num; // 2 + mContext->m_net_channel = input_img->get_shape()->dims[1]; + mContext->net_h = input_img->get_shape()->dims[2]; + mContext->net_w = input_img->get_shape()->dims[3]; + + // 3. get output + mContext->output_num = mContext->bmNetwork->outputTensorNum(); + mContext->min_dim = mContext->bmNetwork->outputTensor(0)->get_shape()->num_dims; + + // 4.converto + float input_scale = input_ratio->get_scale(); + // 归一化 [0-255] -> [0-1] 归一化：y = (1/255) * x + 0 标准化：z = (1/std) * y + (-mean/std) + // z = (1/std) * [(1/255) * x + 0] + (-mean/std) = (1/(255 * std)) * x + (-mean/std) + mContext->converto_attr.alpha_0 = input_scale/(255.f*mContext->stdd[0]); + mContext->converto_attr.beta_0 = -(mContext->mean[0]) / (mContext->stdd[0]); + mContext->converto_attr.alpha_1 = input_scale/(255.f*mContext->stdd[1]); + mContext->converto_attr.beta_1 = -(mContext->mean[1]) / (mContext->stdd[1]); + mContext->converto_attr.alpha_2 = input_scale/(255.f*mContext->stdd[2]); + mContext->converto_attr.beta_2 = -(mContext->mean[2]) / (mContext->stdd[2]); + + } while (false); + return common::ErrorCode::SUCCESS; + } + + common::ErrorCode Ppyoloe_plus::initInternal(const std::string& json) { + common::ErrorCode errorCode = common::ErrorCode::SUCCESS; + do { + // json是否正确 + auto configure = nlohmann::json::parse(json, nullptr, false); + if (!configure.is_object()) { + errorCode = common::ErrorCode::PARSE_CONFIGURE_FAIL; + break; + } + + auto stageNameIt = configure.find(CONFIG_INTERNAL_STAGE_NAME_FIELD); + if (configure.end() != stageNameIt && stageNameIt->is_array()) { + std::vector stages = + stageNameIt->get>(); + if (std::find(stages.begin(), stages.end(), "pre") != stages.end()) { + use_pre = true; + mFpsProfilerName = "fps_ppyoloe_plus_pre"; + } + if (std::find(stages.begin(), stages.end(), "infer") != stages.end()) { + use_infer = true; + mFpsProfilerName = "fps_ppyoloe_plus_infer"; + } + if (std::find(stages.begin(), stages.end(), "post") != stages.end()) { + use_post = true; + mFpsProfilerName = "fps_ppyoloe_plus_post"; + } + + mFpsProfiler.config(mFpsProfilerName, 100); + } + // 新建context,预处理,推理和后处理对象 + mContext = std::make_shared(); + mPreProcess = std::make_shared(); + mInference = std::make_shared(); + mPostProcess = std::make_shared(); + + if (!mPreProcess || !mInference || !mPostProcess || !mContext) { + break; + } + + mContext->deviceId = getDeviceId(); + initContext(configure.dump()); + // 前处理初始化 + mPreProcess->init(mContext); + // 推理初始化 + mInference->init(mContext); + // 后处理初始化 + mPostProcess->init(mContext); + + } while (false); + return errorCode; + } + + void Ppyoloe_plus::process(common::ObjectMetadatas & objectMetadatas) { + common::ErrorCode errorCode = common::ErrorCode::SUCCESS; + if (use_pre) { + IVS_INFO("preprocess"); + errorCode = mPreProcess->preProcess(mContext, objectMetadatas); + if (common::ErrorCode::SUCCESS != errorCode) { + for (unsigned i = 0; i < objectMetadatas.size(); i++) { + objectMetadatas[i]->mErrorCode = errorCode; + } + return; + } + } + // 推理 + if (use_infer) { + IVS_INFO("infer"); + errorCode = mInference->predict(mContext, objectMetadatas); + if (common::ErrorCode::SUCCESS != errorCode) { + for (unsigned i = 0; i < objectMetadatas.size(); i++) { + objectMetadatas[i]->mErrorCode = errorCode; + } + return; + } + } + // 后处理 + if (use_post) { + IVS_INFO("postProcess"); + mPostProcess->postProcess(mContext, objectMetadatas); + } + + } + + common::ErrorCode Ppyoloe_plus::doWork(int dataPipeId) { + common::ErrorCode errorCode = common::ErrorCode::SUCCESS; + + common::ObjectMetadatas objectMetadatas; + std::vector inputPorts = getInputPorts(); + int inputPort = inputPorts[0]; + int outputPort = 0; + if (!getSinkElementFlag()) { + std::vector outputPorts = getOutputPorts(); + outputPort = outputPorts[0]; + } + + common::ObjectMetadatas pendingObjectMetadatas; + + while (objectMetadatas.size() < mContext->max_batch && + (getThreadStatus() == ThreadStatus::RUN)) { + // 如果队列为空则等待 + auto data = popInputData(inputPort, dataPipeId); + if (!data) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + + auto objectMetadata = + std::static_pointer_cast(data); + if (!objectMetadata->mFilter) objectMetadatas.push_back(objectMetadata); + + pendingObjectMetadatas.push_back(objectMetadata); + + if (objectMetadata->mFrame->mEndOfStream) { + break; + } + } + + process(objectMetadatas); + + for (auto& objectMetadata : pendingObjectMetadatas) { + int channel_id_internal = objectMetadata->mFrame->mChannelIdInternal; + int outDataPipeId = + getSinkElementFlag() + ? 0 + : (channel_id_internal % getOutputConnectorCapacity(outputPort)); + errorCode = + pushOutputData(outputPort, outDataPipeId, + std::static_pointer_cast(objectMetadata)); + if (common::ErrorCode::SUCCESS != errorCode) { + IVS_WARN( + "Send data fail, element id: {0:d}, output port: {1:d}, data: " + "{2:p}", + getId(), outputPort, static_cast(objectMetadata.get())); + } + } + mFpsProfiler.add(objectMetadatas.size()); + + return common::ErrorCode::SUCCESS; + } + + void Ppyoloe_plus::setStage(bool pre, bool infer, bool post) { + use_pre = pre; + use_infer = infer; + use_post = post; + } + + void Ppyoloe_plus::initProfiler(std::string name, int interval) { + mFpsProfiler.config(name, 100); + } + + void Ppyoloe_plus::setContext( + std::shared_ptr<::sophon_stream::element::Context> context) { + // check + mContext = std::dynamic_pointer_cast(context); + } + + void Ppyoloe_plus::setPreprocess( + std::shared_ptr<::sophon_stream::element::PreProcess> pre) { + mPreProcess = std::dynamic_pointer_cast(pre); + } + + void Ppyoloe_plus::setInference( + std::shared_ptr<::sophon_stream::element::Inference> infer) { + mInference = std::dynamic_pointer_cast(infer); + } + + void Ppyoloe_plus::setPostprocess( + std::shared_ptr<::sophon_stream::element::PostProcess> post) { + mPostProcess = std::dynamic_pointer_cast(post); + } + + REGISTER_WORKER("ppyoloe_plus", Ppyoloe_plus) + REGISTER_GROUP_WORKER("ppyoloe_plus_group", + sophon_stream::framework::Group, Ppyoloe_plus) +} // namespace ppyoloe_plus +} // namespace element +} // namespace sophon_stream diff --git a/element/algorithm/ppyoloe_plus/src/ppyoloe_plus_inference.cc b/element/algorithm/ppyoloe_plus/src/ppyoloe_plus_inference.cc new file mode 100644 index 0000000..42f5980 --- /dev/null +++ b/element/algorithm/ppyoloe_plus/src/ppyoloe_plus_inference.cc @@ -0,0 +1,50 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (C) 2022 Sophgo Technologies Inc. All rights reserved. +// +// SOPHON-STREAM is licensed under the 2-Clause BSD License except for the +// third-party components. +// +//===----------------------------------------------------------------------===// + +#include "ppyoloe_plus_inference.h" + +namespace sophon_stream { +namespace element { +namespace ppyoloe_plus { + +Ppyoloe_plusInference::~Ppyoloe_plusInference() {} + +void Ppyoloe_plusInference::init(std::shared_ptr context) {} + +common::ErrorCode Ppyoloe_plusInference::predict( + std::shared_ptr context, + common::ObjectMetadatas& objectMetadatas) { + if (objectMetadatas.size() == 0) return common::ErrorCode::SUCCESS; + + if (context->max_batch > 1) { + auto inputTensors = mergeInputDeviceMem(context, objectMetadatas); + auto outputTensors = getOutputDeviceMem(context); + + int ret = 0; + ret = context->bmNetwork->forward(inputTensors->tensors, + outputTensors->tensors); + + splitOutputMemIntoObjectMetadatas(context, objectMetadatas, outputTensors); + } else { + objectMetadatas[0]->mOutputBMtensors = getOutputDeviceMem(context); + int ret = context->bmNetwork->forward( + objectMetadatas[0]->mInputBMtensors->tensors, + objectMetadatas[0]->mOutputBMtensors->tensors); + } + + for(auto obj : objectMetadatas) { + obj->mInputBMtensors = nullptr; + } + + return common::ErrorCode::SUCCESS; +} + +} // namespace ppyoloe_plus +} // namespace element +} // namespace sophon_stream \ No newline at end of file diff --git a/element/algorithm/ppyoloe_plus/src/ppyoloe_plus_post_process.cc b/element/algorithm/ppyoloe_plus/src/ppyoloe_plus_post_process.cc new file mode 100644 index 0000000..36b3274 --- /dev/null +++ b/element/algorithm/ppyoloe_plus/src/ppyoloe_plus_post_process.cc @@ -0,0 +1,165 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (C) 2022 Sophgo Technologies Inc. All rights reserved. +// +// SOPHON-STREAM is licensed under the 2-Clause BSD License except for the +// third-party components. +// +//===----------------------------------------------------------------------===// + +#include "ppyoloe_plus_post_process.h" + +namespace sophon_stream { +namespace element { +namespace ppyoloe_plus { + +void Ppyoloe_plusPostProcess::init(std::shared_ptr context) {} + +void Ppyoloe_plusPostProcess::postProcess(std::shared_ptr context, + common::ObjectMetadatas& objectMetadatas) { + if (objectMetadatas.size() == 0) return; + // write your post process here + + ppYoloePlusBoxVec yolobox_vec; + for (auto obj : objectMetadatas) { + // 每帧处理逻辑 + if (obj->mFrame->mEndOfStream) break; + + // ----收集所有输出结果, ppyoloe有两个输出结果 + std::vector> outputTensors(context->output_num); + for (int i = 0; i < context->output_num; i++) { + outputTensors[i] = std::make_shared( + obj->mOutputBMtensors->handle, + context->bmNetwork->m_netinfo->output_names[i], + context->bmNetwork->m_netinfo->output_scales[i], + obj->mOutputBMtensors->tensors[i].get(), context->bmNetwork->is_soc); + } + + // ---置信度过滤 + yolobox_vec.clear(); + auto out_score = outputTensors[1]; + auto out_coordinate = outputTensors[0]; + int m_class_num = out_score->get_shape()->dims[1]; + + // float* output_data = nullptr; + float *out_score_ptr = nullptr; + float *out_coordinate_ptr = nullptr; + + // box_num = 8400 + int box_num = out_coordinate->get_shape()->dims[1]; + out_score_ptr = (float*)out_score->get_cpu_data(); + out_coordinate_ptr = (float*)out_coordinate->get_cpu_data(); + + + // 计算缩放比例，dw、dh + bm_image frame = *obj->mFrame->mSpData; + int frame_width = frame.width; + int frame_height = frame.height; + int dw = 0, dh = 0; + bool is_align_width = false; + float ratio = get_aspect_scaled_ratio(frame.width, frame.height, context->net_w, context->net_h, &is_align_width); + if (is_align_width) { + dh = (int)((context->net_h - (int)(frame_height*ratio)) / 2); + }else{ + dw = (int)((context->net_w - (int)(frame_width*ratio)) / 2); + } + + // 每个框做类别解码，置信度过滤，结果保存到yolobox_vec中 + for (int i = 0; i < box_num; i++) { + + float *ptr0 = out_score_ptr + i; + float *ptr1 = out_coordinate_ptr + i*4; + + int class_id = argmax_interval(&ptr0[0],m_class_num,box_num); + + float confidence = ptr0[class_id*box_num]; + + if (confidence > context->thresh_conf_min) { + + ppYoloePlusBox box; + box.x = std::max(int(round(ptr1[0])-int(dw/ratio)), 0); + box.y = std::max(int(round(ptr1[1])-int(dh/ratio)), 0); + int x2 = std::min(int(round(ptr1[2])-int(dw/ratio)), frame_width); + int y2 = std::min(int(round(ptr1[3])-int(dh/ratio)), frame.height); + box.width = x2 -box.x ; + box.height = y2 - box.y; + + box.class_id = class_id; + box.score = confidence; + yolobox_vec.push_back(box); + } + } + + // --- NMS---- + NMS(yolobox_vec, context->thresh_nms); + + // ----结果保存到 mDetectedObjectMetadatas + for (auto bbox : yolobox_vec) { + std::shared_ptr detData = + std::make_shared(); + detData->mBox.mX = bbox.x; + detData->mBox.mY = bbox.y; + detData->mBox.mWidth = bbox.width; + detData->mBox.mHeight = bbox.height; + detData->mScores.push_back(bbox.score); + detData->mTopKLabels.push_back(0); + detData->mClassify = bbox.class_id; + detData->mClassifyName = context->class_names[bbox.class_id]; + + if (context->class_thresh_valid) { + detData->mLabelName = context->class_names[detData->mClassify]; + } + + obj->mDetectedObjectMetadatas.push_back(detData); + } + } +} + + +int Ppyoloe_plusPostProcess::argmax_interval(float *data, int class_num, int box_num){ + float max_value = 0.0; + int max_index = 0; + for (int i = 0; i < class_num; ++i) { + float value = data[i*box_num]; + if (value > max_value) { + max_value = value; + max_index = i; + } + } + return max_index; +} + +void Ppyoloe_plusPostProcess::NMS(ppYoloePlusBoxVec& dets, float nmsConfidence) { + int length = dets.size(); + int index = length - 1; + + std::sort(dets.begin(), dets.end(), [](const ppYoloePlusBox& a, const ppYoloePlusBox& b) { return a.score < b.score; }); + + std::vector areas(length); + for (int i = 0; i < length; i++) { + areas[i] = dets[i].width * dets[i].height; + } + + while (index > 0) { + int i = 0; + while (i < index) { + float left = std::max(dets[index].x, dets[i].x); + float top = std::max(dets[index].y, dets[i].y); + float right = std::min(dets[index].x + dets[index].width, dets[i].x + dets[i].width); + float bottom = std::min(dets[index].y + dets[index].height, dets[i].y + dets[i].height); + float overlap = std::max(0.0f, right - left) * std::max(0.0f, bottom - top); + if (overlap / (areas[index] + areas[i] - overlap) > nmsConfidence) { + areas.erase(areas.begin() + i); + dets.erase(dets.begin() + i); + index--; + } else { + i++; + } + } + index--; + } +} + +} // namespace ppyoloe_plus +} // namespace element +} // namespace sophon_stream \ No newline at end of file diff --git a/element/algorithm/ppyoloe_plus/src/ppyoloe_plus_pre_process.cc b/element/algorithm/ppyoloe_plus/src/ppyoloe_plus_pre_process.cc new file mode 100644 index 0000000..1f1cfd4 --- /dev/null +++ b/element/algorithm/ppyoloe_plus/src/ppyoloe_plus_pre_process.cc @@ -0,0 +1,208 @@ +//===----------------------------------------------------------------------===// +// +// Copyright (C) 2022 Sophgo Technologies Inc. All rights reserved. +// +// SOPHON-STREAM is licensed under the 2-Clause BSD License except for the +// third-party components. +// +//===----------------------------------------------------------------------===// + +#include "ppyoloe_plus_pre_process.h" + +namespace sophon_stream { +namespace element { +namespace ppyoloe_plus { + +void Ppyoloe_plusPreProcess::init(std::shared_ptr context) {} + +common::ErrorCode Ppyoloe_plusPreProcess::preProcess( + std::shared_ptr context, + common::ObjectMetadatas& objectMetadatas) { + if (objectMetadatas.size() == 0) return common::ErrorCode::SUCCESS; + initTensors(context, objectMetadatas); + + // write your pre process here + auto jsonPlanner = context->bgr2rgb ? FORMAT_RGB_PLANAR : FORMAT_BGR_PLANAR; + int i = 0; + for (auto& objMetadata : objectMetadatas) { + if (objMetadata->mFrame->mSpData == nullptr) continue; + + // 1、图片预处理，准备模型第一个输入张量 + bm_image image0 = *objMetadata->mFrame->mSpData; + bm_image rgb_img; // rgb图片 + bm_image image_aligned; // 对齐图片 + bm_image resized_img; // 等比缩放图片 + bm_image converto_img; // 归一化图片 + + // convert to RGB_PLANAR + if (image0.image_format != jsonPlanner) { + bm_image_create(context->handle, image0.height, image0.width, jsonPlanner, + image0.data_type, &rgb_img); + auto ret = bm_image_alloc_dev_mem_heap_mask(rgb_img, STREAM_VPU_HEAP_MASK); + STREAM_CHECK(ret == 0, "Alloc Device Memory Failed! Program Terminated.") + bmcv_image_storage_convert(context->handle, 1, &image0, &rgb_img); + } else { + rgb_img = image0; + } + + // 对齐 + bool need_copy = rgb_img.width & (64 - 1); + if (need_copy) { + int stride1[3], stride2[3]; + bm_image_get_stride(rgb_img, stride1); + stride2[0] = FFALIGN(stride1[0], 64); + stride2[1] = FFALIGN(stride1[1], 64); + stride2[2] = FFALIGN(stride1[2], 64); + bm_image_create(context->bmContext->handle(), rgb_img.height, rgb_img.width, + rgb_img.image_format, rgb_img.data_type, &image_aligned, + stride2); + + auto ret = bm_image_alloc_dev_mem_heap_mask(image_aligned, STREAM_VPU_HEAP_MASK); + STREAM_CHECK(ret == 0, "Alloc Device Memory Failed! Program Terminated.") + bmcv_copy_to_atrr_t copyToAttr; + memset(©ToAttr, 0, sizeof(copyToAttr)); + copyToAttr.start_x = 0; + copyToAttr.start_y = 0; + copyToAttr.if_padding = 1; + bmcv_image_copy_to(context->bmContext->handle(), copyToAttr, rgb_img, + image_aligned); + } else { + image_aligned = rgb_img; + } + + // ----等比缩放填充操作 letterbox--- + int aligned_net_w = FFALIGN(context->net_w, 64); + int strides[3] = {aligned_net_w, aligned_net_w, aligned_net_w}; + bm_image_create(context->handle, context->net_h, context->net_w, + jsonPlanner, DATA_TYPE_EXT_1N_BYTE, &resized_img, strides); + auto ret = bm_image_alloc_dev_mem_heap_mask(resized_img, STREAM_VPP_HEAP_MASK); + STREAM_CHECK(ret == 0, "Alloc Device Memory Failed! Program Terminated.") + + bool isAlignWidth = false; // 是否是按宽度缩放比例缩放 + float ratio = get_aspect_scaled_ratio(image_aligned.width, image_aligned.height, context->net_w, context->net_h, &isAlignWidth); + bmcv_padding_atrr_t padding_attr; + memset(&padding_attr, 0, sizeof(padding_attr)); + padding_attr.dst_crop_sty = 0; + padding_attr.dst_crop_stx = 0; + padding_attr.padding_b = 114; + padding_attr.padding_g = 114; + padding_attr.padding_r = 114; + padding_attr.if_memset = 1; + if (isAlignWidth) { + padding_attr.dst_crop_h = image_aligned.height*ratio; // 高度按比例缩放 + padding_attr.dst_crop_w = context->net_w; // 宽度等于网络输入宽 + + int ty1 = (int)((context->net_h - padding_attr.dst_crop_h) / 2); + padding_attr.dst_crop_sty = ty1; // 垂直居中 + padding_attr.dst_crop_stx = 0; // 水平靠左 + }else{ + padding_attr.dst_crop_h = context->net_h; + padding_attr.dst_crop_w = image_aligned.width*ratio; + + int tx1 = (int)((context->net_w - padding_attr.dst_crop_w) / 2); + padding_attr.dst_crop_sty = 0; + padding_attr.dst_crop_stx = tx1; + } + + bmcv_rect_t crop_rect{0, 0, image_aligned.width, image_aligned.height}; + auto ret1 = bmcv_image_vpp_convert_padding(context->handle, 1, image_aligned, &resized_img, + &padding_attr, &crop_rect, BMCV_INTER_NEAREST); + assert(BM_SUCCESS == ret1); + + if(need_copy) bm_image_destroy(image_aligned); + if (image0.image_format != FORMAT_BGR_PLANAR) { + bm_image_destroy(rgb_img); + } + + // -----图片归一化----- + bm_image_data_format_ext img_dtype = DATA_TYPE_EXT_FLOAT32; + auto tensor = context->bmNetwork->inputTensor(0); + if (tensor->get_dtype() == BM_INT8) { + img_dtype = DATA_TYPE_EXT_1N_BYTE_SIGNED; + } else if (tensor->get_dtype() == BM_FLOAT16) { + img_dtype = DATA_TYPE_EXT_FP16; + } + + // converto_img分配空间 + bm_image_create(context->handle, context->net_h, context->net_w, + jsonPlanner, img_dtype, &converto_img); + bm_device_mem_t mem; + int size_byte = 0; + bm_image_get_byte_size(converto_img, &size_byte); + ret = bm_malloc_device_byte_heap(context->handle, &mem, STREAM_NPU_HEAP, size_byte); + STREAM_CHECK(ret == 0, "Alloc Device Memory Failed! Program Terminated.") + + bm_image_attach(converto_img, &mem); + + // 图片归一化, 标准化 + bmcv_image_convert_to(context->handle, 1, context->converto_attr, + &resized_img, &converto_img); + + + bm_image_destroy(resized_img); + + // converto_img绑定到objectMetadatas + bm_image_get_device_mem( + converto_img, + &objectMetadatas[i]->mInputBMtensors->tensors[0]->device_mem); + bm_image_detach(converto_img); + bm_image_destroy(converto_img); + + + // 2、--- prepare ratio tensor (robust version) --- + auto net_info = bmrt_get_network_info(context->bmContext->bmrt(), + context->bmContext->network_name(0).c_str()); + + // ensure tensors vector has at least input_num slots + int input_num = context->bmNetwork->m_netinfo->input_num; // or context->bmNetwork->input_num + if ((int)objectMetadatas[i]->mInputBMtensors->tensors.size() < input_num) { + objectMetadatas[i]->mInputBMtensors->tensors.resize(input_num); + } + + // create shared_ptr slot if empty + if (!objectMetadatas[i]->mInputBMtensors->tensors[1]) { + objectMetadatas[i]->mInputBMtensors->tensors[1] = std::make_shared(); + } + + // allocate device mem for ratio (prefer to do this once in init; shown here per-frame for clarity) + bm_device_mem_t dev_mem; + bm_status_t st = bm_malloc_device_byte(context->bmContext->handle(), + &dev_mem, + net_info->max_input_bytes[1]); + STREAM_CHECK(st == 0, "Alloc device mem for ratio failed"); + + // build the bm_tensor_t that runtime expects + auto ratio_tensor = objectMetadatas[i]->mInputBMtensors->tensors[1]; + ratio_tensor->device_mem = dev_mem; // device mem + ratio_tensor->dtype = context->bmNetwork->inputTensor(1)->get_dtype(); + ratio_tensor->st_mode = BM_STORE_1N; + // set shape exactly as net_info expects (dims array must match) + ratio_tensor->shape = {2, { net_info->stages[0].input_shapes[1].dims[0], 2 } }; + + // compute ratio (example: h_ratio, w_ratio OR width/height according to your model) + float ratio_arr[2]; + ratio_arr[0] = ratio; // consistent with your previous code + ratio_arr[1] = ratio; + + // check bytesize expected by runtime for this tensor + size_t expect_bytes = bmrt_tensor_bytesize(ratio_tensor.get()); + if (expect_bytes == 0) { + IVS_CRITICAL("bmrt_tensor_bytesize returned 0 for ratio tensor"); + } + + // copy host -> device to the exact device_mem we assigned + st = bm_memcpy_s2d_partial(context->bmContext->handle(), + ratio_tensor->device_mem, + (void*)ratio_arr, + expect_bytes); + STREAM_CHECK(st == 0, "bm_memcpy_s2d_partial for ratio failed"); + + i++; + } + + return common::ErrorCode::SUCCESS; +} + +} // namespace ppyoloe_plus +} // namespace element +} // namespace sophon_stream \ No newline at end of file