Add RKYOLOv5 RKYOLOX RKYOLOV7 (PaddlePaddle#709)

* 更正代码格式 * 更正代码格式 * 修复语法错误 * fix rk error * update * update * update * update * update * update * update Co-authored-by: Jason <[email protected]>
leyond123 · Dec 10, 2022 · c7dc7d5 · c7dc7d5
1 parent 6f5521e
commit c7dc7d5
Show file tree

Hide file tree

Showing 25 changed files with 1,516 additions and 1 deletion.
diff --git a/examples/vision/detection/paddledetection/rknpu2/README.md b/examples/vision/detection/paddledetection/rknpu2/README.md
@@ -113,5 +113,7 @@ Preprocess:
   type: Resize
 ```
 
+## 其他链接
+- [Cpp部署](./cpp)
 - [Python部署](./python)
 - [视觉模型预测结果](../../../../../docs/api/vision_results/)
diff --git a/examples/vision/detection/rkyolo/README.md b/examples/vision/detection/rkyolo/README.md
@@ -0,0 +1,18 @@
+# RKYOLO准备部署模型
+
+RKYOLO参考[rknn_model_zoo](https://github.com/airockchip/rknn_model_zoo/tree/main/models/CV/object_detection/yolo)的代码
+对RKYOLO系列模型进行了封装，目前支持RKYOLOV5系列模型的部署。
+
+## 支持模型列表
+
+* RKYOLOV5
+
+## 模型转换example
+
+请参考[RKNN_model_convert](https://github.com/airockchip/rknn_model_zoo/tree/main/models/CV/object_detection/yolo/RKNN_model_convert)
+
+
+## 其他链接
+- [Cpp部署](./cpp)
+- [Python部署](./python)
+- [视觉模型预测结果](../../../../docs/api/vision_results/)
diff --git a/examples/vision/detection/rkyolo/cpp/CMakeLists.txt b/examples/vision/detection/rkyolo/cpp/CMakeLists.txt
@@ -0,0 +1,37 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.10)
+project(rknpu2_test)
+
+set(CMAKE_CXX_STANDARD 14)
+
+# 指定下载解压后的fastdeploy库路径
+set(FASTDEPLOY_INSTALL_DIR "thirdpartys/fastdeploy-0.0.3")
+
+include(${FASTDEPLOY_INSTALL_DIR}/FastDeployConfig.cmake)
+include_directories(${FastDeploy_INCLUDE_DIRS})
+
+add_executable(infer_rkyolo infer_rkyolo.cc)
+target_link_libraries(infer_rkyolo ${FastDeploy_LIBS})
+
+
+
+set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/build/install)
+
+install(TARGETS infer_rkyolo DESTINATION ./)
+
+install(DIRECTORY model DESTINATION ./)
+install(DIRECTORY images DESTINATION ./)
+
+file(GLOB FASTDEPLOY_LIBS ${FASTDEPLOY_INSTALL_DIR}/lib/*)
+message("${FASTDEPLOY_LIBS}")
+install(PROGRAMS ${FASTDEPLOY_LIBS} DESTINATION lib)
+
+file(GLOB ONNXRUNTIME_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/onnxruntime/lib/*)
+install(PROGRAMS ${ONNXRUNTIME_LIBS} DESTINATION lib)
+
+install(DIRECTORY ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/opencv/lib DESTINATION ./)
+
+file(GLOB PADDLETOONNX_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/paddle2onnx/lib/*)
+install(PROGRAMS ${PADDLETOONNX_LIBS} DESTINATION lib)
+
+file(GLOB RKNPU2_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/rknpu2_runtime/${RKNN2_TARGET_SOC}/lib/*)
+install(PROGRAMS ${RKNPU2_LIBS} DESTINATION lib)
diff --git a/examples/vision/detection/rkyolo/cpp/README.md b/examples/vision/detection/rkyolo/cpp/README.md
@@ -0,0 +1,69 @@
+# RKYOLO C++部署示例
+
+本目录下提供`infer_xxxxx.cc`快速完成RKYOLO模型在Rockchip板子上上通过二代NPU加速部署的示例。
+
+在部署前，需确认以下两个步骤:
+
+1. 软硬件环境满足要求
+2. 根据开发环境，下载预编译部署库或者从头编译FastDeploy仓库
+
+以上步骤请参考[RK2代NPU部署库编译](../../../../../docs/cn/build_and_install/rknpu2.md)实现
+
+## 生成基本目录文件
+
+该例程由以下几个部分组成
+```text
+.
+├── CMakeLists.txt
+├── build  # 编译文件夹
+├── image  # 存放图片的文件夹
+├── infer_rkyolo.cc
+├── model  # 存放模型文件的文件夹
+└── thirdpartys  # 存放sdk的文件夹
+```
+
+首先需要先生成目录结构
+```bash
+mkdir build
+mkdir images
+mkdir model
+mkdir thirdpartys
+```
+
+## 编译
+
+### 编译并拷贝SDK到thirdpartys文件夹
+
+请参考[RK2代NPU部署库编译](../../../../../../docs/cn/build_and_install/rknpu2.md)仓库编译SDK，编译完成后，将在build目录下生成
+fastdeploy-0.0.3目录，请移动它至thirdpartys目录下.
+
+### 拷贝模型文件，以及配置文件至model文件夹
+在Paddle动态图模型 -> Paddle静态图模型 -> ONNX模型的过程中，将生成ONNX文件以及对应的yaml配置文件，请将配置文件存放到model文件夹内。
+转换为RKNN后的模型文件也需要拷贝至model。
+
+### 准备测试图片至image文件夹
+```bash
+wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
+cp 000000014439.jpg ./images
+```
+
+### 编译example
+
+```bash
+cd build
+cmake ..
+make -j8
+make install
+```
+
+## 运行例程
+
+```bash
+cd ./build/install
+./infer_picodet model/ images/000000014439.jpg
+```
+
+
+- [模型介绍](../../)
+- [Python部署](../python)
+- [视觉模型预测结果](../../../../../../docs/api/vision_results/)
diff --git a/examples/vision/detection/rkyolo/cpp/infer_rkyolo.cc b/examples/vision/detection/rkyolo/cpp/infer_rkyolo.cc
@@ -0,0 +1,53 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "fastdeploy/vision.h"
+
+void RKNPU2Infer(const std::string& model_file, const std::string& image_file) {
+  struct timeval start_time, stop_time;
+
+  auto option = fastdeploy::RuntimeOption();
+  option.UseRKNPU2();
+
+  auto format = fastdeploy::ModelFormat::RKNN;
+
+  auto model = fastdeploy::vision::detection::RKYOLOV5(
+      model_file, option,format);
+
+  auto im = cv::imread(image_file);
+
+  fastdeploy::vision::DetectionResult res;
+  if (!model.Predict(im, &res)) {
+    std::cerr << "Failed to predict." << std::endl;
+    return;
+  }
+  std::cout << res.Str() << std::endl;
+  auto vis_im = fastdeploy::vision::VisDetection(im, res,0.5);
+  cv::imwrite("vis_result.jpg", vis_im);
+  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
+}
+
+int main(int argc, char* argv[]) {
+  if (argc < 3) {
+    std::cout
+        << "Usage: infer_demo path/to/model_dir path/to/image run_option, "
+           "e.g ./infer_model ./picodet_model_dir ./test.jpeg"
+        << std::endl;
+    return -1;
+  }
+
+  RKNPU2Infer(argv[1], argv[2]);
+
+  return 0;
+}
+
diff --git a/examples/vision/detection/rkyolo/python/README.md b/examples/vision/detection/rkyolo/python/README.md
@@ -0,0 +1,34 @@
+# RKYOLO Python部署示例
+
+在部署前，需确认以下两个步骤
+
+- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../../docs/cn/build_and_install/rknpu2.md)
+
+本目录下提供`infer.py`快速完成Picodet在RKNPU上部署的示例。执行如下脚本即可完成
+
+```bash
+# 下载部署示例代码
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy/examples/vision/detection/rkyolo/python
+
+# 下载图片
+wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
+
+# copy model
+cp -r ./model /path/to/FastDeploy/examples/vision/detection/rkyolo/python
+
+# 推理
+python3 infer.py --model_file ./model/  \
+                  --image 000000014439.jpg
+```
+
+
+## 注意事项
+RKNPU上对模型的输入要求是使用NHWC格式，且图片归一化操作会在转RKNN模型时，内嵌到模型中，因此我们在使用FastDeploy部署时，
+
+## 其它文档
+
+- [PaddleDetection 模型介绍](..)
+- [PaddleDetection C++部署](../cpp)
+- [模型预测结果说明](../../../../../../docs/api/vision_results/)
+- [转换PaddleDetection RKNN模型文档](../README.md)
diff --git a/examples/vision/detection/rkyolo/python/infer.py b/examples/vision/detection/rkyolo/python/infer.py
@@ -0,0 +1,53 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import fastdeploy as fd
+import cv2
+import os
+
+
+def parse_arguments():
+    import argparse
+    import ast
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_file", required=True, help="Path of rknn model.")
+    parser.add_argument(
+        "--image", type=str, required=True, help="Path of test image file.")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+
+    model_file = args.model_file
+    params_file = ""
+
+    # 配置runtime，加载模型
+    runtime_option = fd.RuntimeOption()
+    runtime_option.use_rknpu2()
+
+    model = fd.vision.detection.RKYOLOV5(
+        model_file,
+        runtime_option=runtime_option,
+        model_format=fd.ModelFormat.RKNN)
+
+    # 预测图片分割结果
+    im = cv2.imread(args.image)
+    result = model.predict(im)
+    print(result)
+
+    # 可视化结果
+    vis_im = fd.vision.vis_detection(im, result, score_threshold=0.5)
+    cv2.imwrite("visualized_result.jpg", vis_im)
+    print("Visualized result save in ./visualized_result.jpg")
diff --git a/fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.cc b/fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.cc
@@ -190,6 +190,8 @@ bool RKNPU2Backend::GetModelInputOutputInfos() {
       FDERROR << "rknpu2_backend only support input format is NHWC or UNDEFINED" << std::endl;
     }
 
+    DumpTensorAttr(input_attrs_[i]);
+
     // copy input_attrs_ to input tensor info
     std::string temp_name = input_attrs_[i].name;
     std::vector<int> temp_shape{};
@@ -234,6 +236,8 @@ bool RKNPU2Backend::GetModelInputOutputInfos() {
                 << std::endl;
     }
 
+    DumpTensorAttr(output_attrs_[i]);
+
     // copy output_attrs_ to output tensor
     std::string temp_name = output_attrs_[i].name;
     std::vector<int> temp_shape{};
@@ -342,7 +346,6 @@ bool RKNPU2Backend::Infer(std::vector<FDTensor>& inputs,
         return false;
       }
       // default output type is depend on model, this requires float32 to compute top5
-      output_attrs_[i].type = RKNN_TENSOR_FLOAT32;
       ret = rknn_set_io_mem(ctx, output_mems_[i], &output_attrs_[i]);
       // set output memory and attribute
       if (ret != RKNN_SUCC) {
@@ -389,6 +392,8 @@ bool RKNPU2Backend::Infer(std::vector<FDTensor>& inputs,
     }
     (*outputs)[i].Resize(temp_shape, outputs_desc_[i].dtype,
                          outputs_desc_[i].name);
+    std::vector<float>  output_scale = {output_attrs_[i].scale};
+    (*outputs)[i].SetQuantizationInfo(output_attrs_[i].zp, output_scale);
     memcpy((*outputs)[i].MutableData(), (float*)output_mems_[i]->virt_addr,
            (*outputs)[i].Nbytes());
   }

diff --git a/fastdeploy/core/fd_tensor.cc b/fastdeploy/core/fd_tensor.cc
@@ -138,6 +138,11 @@ void FDTensor::Resize(const std::vector<int64_t>& new_shape) {
   external_data_ptr = nullptr;
 }
 
+void FDTensor::SetQuantizationInfo(int32_t zero_point,std::vector<float>& scale){
+  quantized_parameter_.first = zero_point;
+  quantized_parameter_.second = scale;
+}
+
 void FDTensor::Resize(const std::vector<int64_t>& new_shape,
                       const FDDataType& data_type,
                       const std::string& tensor_name,
@@ -450,4 +455,9 @@ FDTensor& FDTensor::operator=(FDTensor&& other) {
   return *this;
 }
 
+const std::pair<int32_t, std::vector<float>>
+FDTensor::GetQuantizationInfo() const{
+  return quantized_parameter_;
+}
+
 }  // namespace fastdeploy
diff --git a/fastdeploy/core/fd_tensor.h b/fastdeploy/core/fd_tensor.h
@@ -25,6 +25,11 @@
 namespace fastdeploy {
 
 struct FASTDEPLOY_DECL FDTensor {
+  // These two parameters are general parameters of quantitative model.
+  std::pair<int32_t, std::vector<float>> quantized_parameter_ = {0, {0}};
+  void SetQuantizationInfo(int32_t zero_point, std::vector<float>& scale);
+  const std::pair<int32_t, std::vector<float>> GetQuantizationInfo() const;
+
   // std::vector<int8_t> data;
   void* buffer_ = nullptr;
   std::vector<int64_t> shape = {0};

diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h
@@ -28,6 +28,7 @@
 #include "fastdeploy/vision/detection/contrib/yolov7end2end_ort.h"
 #include "fastdeploy/vision/detection/contrib/yolov7end2end_trt.h"
 #include "fastdeploy/vision/detection/contrib/yolox.h"
+#include "fastdeploy/vision/detection/contrib/rknpu2/model.h"
 #include "fastdeploy/vision/detection/ppdet/model.h"
 #include "fastdeploy/vision/facealign/contrib/face_landmark_1000.h"
 #include "fastdeploy/vision/facealign/contrib/pfld.h"