diff --git a/cmake/opencv.cmake b/cmake/opencv.cmake index 4d5b18c92c..4ce9058332 100755 --- a/cmake/opencv.cmake +++ b/cmake/opencv.cmake @@ -199,6 +199,6 @@ else() endif() find_package(OpenCV REQUIRED PATHS ${OpenCV_DIR} NO_DEFAULT_PATH) include_directories(${OpenCV_INCLUDE_DIRS}) - list(APPEND DEPEND_LIBS opencv_core opencv_video opencv_highgui opencv_imgproc opencv_imgcodecs) + list(APPEND DEPEND_LIBS opencv_core opencv_video opencv_highgui opencv_imgproc opencv_imgcodecs opencv_calib3d opencv_features2d opencv_flann) endif() endif() diff --git a/docs/api/vision_results/perception_result.md b/docs/api/vision_results/perception_result.md new file mode 100755 index 0000000000..ed891eb556 --- /dev/null +++ b/docs/api/vision_results/perception_result.md @@ -0,0 +1,48 @@ +English | [简体中文](detection_result_CN.md) +# PerceptionResult target detection result + +The PerceptionResult code is defined in `fastdeploy/vision/common/result.h`, which is used to indicate the detected 3D target: two-dimensional target frame, target frame length, width and height, target category and target confidence, target orientation angle and observation angle etc. + +## C++ definition + +```c++ +fastdeploy::vision::PerceptionResult +``` + +```c++ +struct PerceptionResult { + std::vector scores; + std::vector label_ids; + std::vector> boxes; + std::vector> center; + std::vectorobservation_angle; + std::vectoryaw_angle; + std::vector>velocity; + void Clear(); + std::string Str(); +}; +``` + +- **scores**: Member variable, indicating the confidence of all detected targets, `scores.size()` indicates the number of detected boxes +- **label_ids**: Member variable, representing all detected target categories, the number of elements is consistent with `scores.size()` +- **boxes**: Member variable, representing the coordinates of all detected target boxes, the number of elements is consistent with `scores.size()`, and each box represents xmin, ymin, xmax, ymax in turn with 7 float values , h, w, l, the coordinates of the upper left and lower right corners and the length, width and height of the 3D box +- **center**: member variable, indicating the center point coordinates of all detected target frames, the number of elements is consistent with `scores.size()`, and each frame uses 3 float values to represent the center point coordinates of the frame in turn +- **observation_angle**: Member variable, indicating the observation angle of the detected frame, and the number of elements is consistent with `scores.size()` +- **yaw_angle**: Member variable, indicating the orientation angle of the detected frame, the number of elements is consistent with `scores.size()` +- **velocity**: Member variable, indicating the velocity of the detected frame, the number of elements is consistent with `scores.size()` +- **Clear()**: member function, used to clear the results stored in the structure +- **Str()**: member function, output the information in the structure as a string (for Debug) + +## Python definition + +```python +fastdeploy.vision.PerceptionResult +``` + +- **scores**(list of float): Member variable, indicating the confidence of all detected targets, `scores.size()` indicates the number of detected frames +- **label_ids**(list of int): Member variable, representing all detected target categories, the number of elements is consistent with `scores.size()` +- **boxes**(list of list(float)): Member variable, indicating the coordinates of all detected target boxes, the number of elements is the same as `scores.size()`, and each box is in order of 7 float values Indicates xmin, ymin, xmax, ymax, h, w, l, that is, the coordinates of the upper left and lower right corners and the length, width and height of the 3D box +- **center**(list of list(float)): Member variable, which represents the coordinates of the center points of all detected target frames, the number of elements is the same as `scores.size()`, and each frame is represented by 3 floats The values ​​in turn represent the coordinates of the center point of the box +- **observation_angle**: member variable, indicating the orientation angle of the detected frame, and the number of elements is consistent with `scores.size()` +- **yaw_angle**: Member variable, indicating the orientation angle of the detected frame, the number of elements is consistent with `scores.size()` +- **velocity**: Member variable, indicating the velocity of the detected frame, the number of elements is consistent with `scores.size()` diff --git a/docs/api/vision_results/perception_result_CN.md b/docs/api/vision_results/perception_result_CN.md new file mode 100755 index 0000000000..8df7a24891 --- /dev/null +++ b/docs/api/vision_results/perception_result_CN.md @@ -0,0 +1,48 @@ +简体中文 ｜ [English](perception_result.md) +# PerceptionResult 目标检测结果 + +PerceptionResult`fastdeploy/vision/common/result.h`中，用于表明检测出来的3D目标的：二维目标框、目标框长宽高、目标类别和目标置信度、目标朝向角和观测角等。 + +## C++ 定义 + +```c++ +fastdeploy::vision::PerceptionResult +``` + +```c++ +struct PerceptionResult { + std::vector scores; + std::vector label_ids; + std::vector> boxes; + std::vector> center; + std::vectorobservation_angle; + std::vectoryaw_angle; + std::vector>velocity; + void Clear(); + std::string Str(); +}; +``` + +- **scores**: 成员变量，表示检测出来的所有目标置信度，`scores.size()`表示检测出来框的个数 +- **label_ids**: 成员变量，表示检测出来的所有目标类别，其元素个数与`scores.size()`一致 +- **boxes**: 成员变量，表示检测出来的所有目标框坐标，其元素个数与`scores.size()`一致，每个框以7个float数值依次表示xmin, ymin, xmax, ymax，h, w, l， 即左上角和右下角坐标以及3D框的长宽高 +- **center**: 成员变量，表示检测出来的所有目标框中心点坐标，其元素个数与`scores.size()`一致，每个框以3个float数值依次表示框中心点坐标 +- **observation_angle**: 成员变量，表示检测出来的框的观测角，其元素个数与`scores.size()`一致 +- **yaw_angle**: 成员变量，表示检测出来的框的朝向角，其元素个数与`scores.size()`一致 +- **velocity**: 成员变量，表示检测出来的框的速度，其元素个数与`scores.size()`一致 +- **Clear()**: 成员函数，用于清除结构体中存储的结果 +- **Str()**: 成员函数，将结构体中的信息以字符串形式输出（用于Debug） + +## Python 定义 + +```python +fastdeploy.vision.PerceptionResult +``` + +- **scores**(list of float): 成员变量，表示检测出来的所有目标置信度，`scores.size()`表示检测出来框的个数 +- **label_ids**(list of int): 成员变量，表示检测出来的所有目标类别，其元素个数与`scores.size()`一致 +- **boxes**(list of list(float)): 成员变量，表示检测出来的所有目标框坐标，其元素个数与`scores.size()`一致，每个框以7个float数值依次表示xmin, ymin, xmax, ymax，h, w, l， 即左上角和右下角坐标以及3D框的长宽高 +- **center**(list of list(float)): 成员变量，表示检测出来的所有目标框中心点坐标，其元素个数与`scores.size()`一致，每个框以3个float数值依次表示框中心点坐标 +- **observation_angle**: 成员变量，表示检测出来的框的朝向角，其元素个数与`scores.size()`一致 +- **yaw_angle**: 成员变量，表示检测出来的框的朝向角，其元素个数与`scores.size()`一致 +- **velocity**: 成员变量，表示检测出来的框的速度，其元素个数与`scores.size()`一致 diff --git a/examples/vision/perception/paddle3d/smoke/README.md b/examples/vision/perception/paddle3d/smoke/README.md new file mode 100755 index 0000000000..a505953d2a --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/README.md @@ -0,0 +1,11 @@ +English | [简体中文](README_CN.md) + +# Smoke Ready-to-deploy Model + +The Smoke deployment model implements the Smoke model from Paddle3D. For more detailed information about the model, please refer to [Smoke Introduction](https://github.com/PaddlePaddle/Paddle3D/tree/develop/docs/models/smoke) + + +## Detailed Deployment Documents + +- [Python Deployment](python) +- [C++ Deployment](cpp) diff --git a/examples/vision/perception/paddle3d/smoke/README_CN.md b/examples/vision/perception/paddle3d/smoke/README_CN.md new file mode 100755 index 0000000000..c0e75aade2 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/README_CN.md @@ -0,0 +1,12 @@ +[English](README.md) | 简体中文 + +# Smoke 准备部署模型 + +Smoke 部署模型实现来自 Paddle3D 的 Smoke 模型，模型相关的更多详细信息可以参考[Smoke 介绍](https://github.com/PaddlePaddle/Paddle3D/tree/develop/docs/models/smoke) + + + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/perception/paddle3d/smoke/cpp/CMakeLists.txt b/examples/vision/perception/paddle3d/smoke/cpp/CMakeLists.txt new file mode 100755 index 0000000000..93540a7e83 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/perception/paddle3d/smoke/cpp/README.md b/examples/vision/perception/paddle3d/smoke/cpp/README.md new file mode 100755 index 0000000000..02203797db --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/cpp/README.md @@ -0,0 +1,80 @@ +English | [简体中文](README_CN.md) +# Smoke C++ Deployment Example + +This directory provides an example of `infer.cc` to quickly complete the deployment of Smoke on CPU/GPU. + +Before deployment, the following two steps need to be confirmed + +- 1. The hardware and software environment meets the requirements, refer to [FastDeploy environment requirements](../../../../../docs/en/build_and_install/download_prebuilt_libraries.md) +- 2. According to the development environment, download the precompiled deployment library and samples code, refer to [FastDeploy prebuilt library](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +Taking CPU inference on Linux as an example, execute the following command in this directory to complete the compilation test. To support this model, you need to ensure FastDeploy version 1.0.6 or higher (x.x.x>=1.0.6) + +```bash +mkdir build +cd build +# Download the FastDeploy precompiled library, users can choose the appropriate version to use in the `FastDeploy precompiled library` mentioned above +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-x.x.x.tgz +tar xvf fastdeploy-linux-x64-x.x.x.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-x.x.x +make -j + +wget https://bj.bcebos.com/fastdeploy/models/smoke.tar.gz +tar -xf smoke.tar.gz +wget https://bj.bcebos.com/fastdeploy/models/smoke_test.png + +# CPU +./infer_demo smoke smoke_test.png 0 +# GPU +./infer_demo smoke smoke_test.png 1 + +``` + +The visual result after running is shown in the figure below + + + +The above commands are only applicable to Linux or MacOS. For the usage of SDK under Windows, please refer to: +- [How to use FastDeploy C++ SDK in Windows](../../../../../docs/en/faq/use_sdk_on_windows.md) + +## Smoke C++ interface + +### Class Smoke + +```c++ +fastdeploy::vision::detection::Smoke( + const string& model_file, + const string& params_file, + const string& config_file, + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) +``` + +Smoke model loading and initialization. + +**parameter** + +> * **model_file**(str): model file path +> * **params_file**(str): parameter file path +> * **config_file**(str): configuration file path +> * **runtime_option**(RuntimeOption): Backend reasoning configuration, the default is None, that is, the default configuration is used +> * **model_format**(ModelFormat): model format, the default is Paddle format + +#### Predict function + +> ```c++ +> Smoke::Predict(cv::Mat* im, PerceptionResult* result) +> ``` +> +> Model prediction interface, the input image directly outputs the detection result. +> +> **parameters** +> +> > * **im**: input image, note that it must be in HWC, BGR format +> > * **result**: Detection result, including the detection frame, the confidence of each frame, PerceptionResult description reference [visual model prediction results](../../../../../docs/api /vision_results/) + + +- [Model Introduction](../../) +- [Python deployment](../python) +- [Vision Model Prediction Results](../../../../../docs/api/vision_results/) +- [How to switch model inference backend engine](../../../../../docs/en/faq/how_to_change_backend.md) diff --git a/examples/vision/perception/paddle3d/smoke/cpp/README_CN.md b/examples/vision/perception/paddle3d/smoke/cpp/README_CN.md new file mode 100755 index 0000000000..47ea955543 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/cpp/README_CN.md @@ -0,0 +1,80 @@ +[English](README.md) | 简体中文 +# Smoke C++部署示例 + +本目录下提供 `infer.cc` 快速完成 Smoke 在 CPU/GPU 上部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +以Linux上 CPU 推理为例，在本目录执行如下命令即可完成编译测试，支持此模型需保证 FastDeploy 版本1.0.6以上(x.x.x>=1.0.6) + +```bash +mkdir build +cd build +# 下载FastDeploy预编译库，用户可在上文提到的`FastDeploy预编译库`中自行选择合适的版本使用 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-x.x.x.tgz +tar xvf fastdeploy-linux-x64-x.x.x.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-x.x.x +make -j + +wget https://bj.bcebos.com/fastdeploy/models/smoke.tar.gz +tar -xf smoke.tar.gz +wget https://bj.bcebos.com/fastdeploy/models/smoke_test.png + +# CPU推理 +./infer_demo smoke smoke_test.png 0 +# GPU推理 +./infer_demo smoke smoke_test.png 1 + +``` + +运行完成可视化结果如下图所示 + + + +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## Smoke C++ 接口 + +### Smoke 类 + +```c++ +fastdeploy::vision::detection::Smoke( + const string& model_file, + const string& params_file, + const string& config_file, + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) +``` + +Smoke模型加载和初始化。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **config_file**(str): 配置文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +#### Predict函数 + +> ```c++ +> Smoke::Predict(cv::Mat* im, PerceptionResult* result) +> ``` +> +> 模型预测接口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **im**: 输入图像，注意需为HWC，BGR格式 +> > * **result**: 检测结果，包括检测框，各个框的置信度, PerceptionResult 说明参考[视觉模型预测结果](../../../../../docs/api/vision_results/) + + +- [模型介绍](../../) +- [Python部署](../python) +- [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/perception/paddle3d/smoke/cpp/infer.cc b/examples/vision/perception/paddle3d/smoke/cpp/infer.cc new file mode 100644 index 0000000000..9506e33e1f --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/cpp/infer.cc @@ -0,0 +1,83 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void InitAndInfer(const std::string& model_dir, const std::string& image_file, + const fastdeploy::RuntimeOption& option) { + auto model_file = model_dir + sep + "smoke.pdmodel"; + auto params_file = model_dir + sep + "smoke.pdiparams"; + auto config_file = model_dir + sep + "infer_cfg.yml"; + fastdeploy::vision::EnableFlyCV(); + auto model = fastdeploy::vision::perception::Smoke( + model_file, params_file, config_file, option, + fastdeploy::ModelFormat::PADDLE); + assert(model.Initialized()); + + auto im = cv::imread(image_file); + + fastdeploy::vision::PerceptionResult res; + if (!model.Predict(im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + std::cout << res.Str() << std::endl; + + auto vis_im = fastdeploy::vision::VisPerception(im, res, config_file); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + std::cout << "Usage: infer_demo path/to/paddle_model" + "path/to/image " + "run_option, " + "e.g ./infer_demo ./smoke ./00000.png 0" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with paddle-trt" + << std::endl; + return -1; + } + + fastdeploy::RuntimeOption option; + if (std::atoi(argv[3]) == 0) { + option.UseCpu(); + } else if (std::atoi(argv[3]) == 1) { + option.UseGpu(); + } else if (std::atoi(argv[3]) == 2) { + option.UseGpu(); + option.UseTrtBackend(); + option.EnablePaddleToTrt(); + option.SetTrtInputShape("images", {1, 3, 384, 1280}); + option.SetTrtInputShape("down_ratios", {1, 2}); + option.SetTrtInputShape("trans_cam_to_img", {1, 3, 3}); + option.SetTrtInputData("trans_cam_to_img", + {721.53771973, 0., 609.55932617, 0., 721.53771973, + 172.85400391, 0, 0, 1}); + option.EnablePaddleTrtCollectShape(); + } + option.UsePaddleBackend(); + + std::string model_dir = argv[1]; + std::string test_image = argv[2]; + InitAndInfer(model_dir, test_image, option); + return 0; +} diff --git a/examples/vision/perception/paddle3d/smoke/python/README.md b/examples/vision/perception/paddle3d/smoke/python/README.md new file mode 100755 index 0000000000..7ff468caa9 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/python/README.md @@ -0,0 +1,67 @@ +English | [简体中文](README_CN.md) +# Smoke Python Deployment Example + +Before deployment, the following two steps need to be confirmed + +- 1. The hardware and software environment meets the requirements, refer to [FastDeploy environment requirements](../../../../../docs/en/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl package installation, refer to [FastDeploy Python Installation](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +This directory provides an example of `infer.py` to quickly complete the deployment of Smoke on CPU/GPU. Execute the following script to complete + +```bash +#Download deployment sample code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd examples/vision/vision/paddle3d/smoke/python + +wget https://bj.bcebos.com/fastdeploy/models/smoke.tar.gz +tar -xf smoke.tar.gz +wget https://bj.bcebos.com/fastdeploy/models/smoke_test.png + +# CPU reasoning +python infer.py --model smoke --image smoke_test.png --device cpu +# GPU inference +python infer.py --model smoke --image smoke_test.png --device gpu +``` + +The visual result after running is shown in the figure below + + + +## Smoke Python interface + +```python +fastdeploy.vision.detection.Smoke(model_file, params_file, config_file, runtime_option=None, model_format=ModelFormat.PADDLE) +``` + +Smoke model loading and initialization. + +**parameter** +> * **model_file**(str): model file path +> * **params_file**(str): parameter file path +> * **config_file**(str): configuration file path +> * **runtime_option**(RuntimeOption): Backend reasoning configuration, the default is None, that is, the default configuration is used +> * **model_format**(ModelFormat): model format, the default is Paddle format + +### predict function + +> ```python +> Smoke. predict(image_data) +> ``` +> +> Model prediction interface, the input image directly outputs the detection result. +> +> **parameters** +> +> > * **image_data**(np.ndarray): input data, note that it must be in HWC, BGR format + +> **Back** +> +> > Return the `fastdeploy.vision.PerceptionResult` structure, structure description reference document [Vision Model Prediction Results](../../../../../docs/api/vision_results/) + + +## Other documents + +- [Smoke Model Introduction](..) +- [Smoke C++ deployment](../cpp) +- [Description of model prediction results](../../../../../docs/api/vision_results/) +- [How to switch model inference backend engine](../../../../../docs/en/faq/how_to_change_backend.md) diff --git a/examples/vision/perception/paddle3d/smoke/python/README_CN.md b/examples/vision/perception/paddle3d/smoke/python/README_CN.md new file mode 100755 index 0000000000..5bf072f91a --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/python/README_CN.md @@ -0,0 +1,68 @@ +[English](README.md) | 简体中文 + +# Smoke Python 部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl 包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供 `infer.py` 快速完成 Smoke 在 CPU/GPU上部署的示例。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd examples/vision/vision/paddle3d/smoke/python + +wget https://bj.bcebos.com/fastdeploy/models/smoke.tar.gz +tar -xf smoke.tar.gz +wget https://bj.bcebos.com/fastdeploy/models/smoke_test.png + +# CPU推理 +python infer.py --model smoke --image smoke_test.png --device cpu +# GPU推理 +python infer.py --model smoke --image smoke_test.png --device gpu +``` + +运行完成可视化结果如下图所示 + + + +## Smoke Python接口 + +```python +fastdeploy.vision.detection.Smoke(model_file, params_file, config_file, runtime_option=None, model_format=ModelFormat.PADDLE) +``` + +Smoke模型加载和初始化。 + +**参数** +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **config_file**(str): 配置文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +### predict 函数 + +> ```python +> Smoke.predict(image_data) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **image_data**(np.ndarray): 输入数据，注意需为HWC，BGR格式 + +> **返回** +> +> > 返回`fastdeploy.vision.PerceptionResult`结构体，结构体说明参考文档[视觉模型预测结果](../../../../../docs/api/vision_results/) + + +## 其它文档 + +- [Smoke 模型介绍](..) +- [Smoke C++部署](../cpp) +- [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/perception/paddle3d/smoke/python/infer.py b/examples/vision/perception/paddle3d/smoke/python/infer.py new file mode 100755 index 0000000000..8a656ac5f1 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/python/infer.py @@ -0,0 +1,50 @@ +import fastdeploy as fd +import cv2 +import os +from fastdeploy import ModelFormat + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", required=True, help="Path of smoke paddle model.") + parser.add_argument( + "--image", required=True, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "gpu": + option.use_gpu(0) + if args.device.lower() == "cpu": + option.use_cpu() + return option + + +args = parse_arguments() + +model_file = os.path.join(args.model, "smoke.pdmodel") +params_file = os.path.join(args.model, "smoke.pdiparams") +config_file = os.path.join(args.model, "infer_cfg.yml") +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.perception.Smoke( + model_file, params_file, config_file, runtime_option=runtime_option) + +# 预测图片检测结果 +im = cv2.imread(args.image) +result = model.predict(im) +print(result) + +# 预测结果可视化 +vis_im = fd.vision.vis_perception(im, result, config_file) +cv2.imwrite("visualized_result.jpg", vis_im) +print("Visualized result save in ./visualized_result.jpg") diff --git a/examples/vision/perception/paddle3d/smoke/serving/README.md b/examples/vision/perception/paddle3d/smoke/serving/README.md new file mode 100755 index 0000000000..f2219cbc37 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/README.md @@ -0,0 +1,85 @@ +English | [简体中文](README_CN.md) +# Smoke service deployment example + +This document introduces the service deployment of the Smoke model in Paddle3D. + +For Smoke model export and pre-training model download, please refer to [Smoke Model Deployment](../README.md) document. + +Before service deployment, you need to confirm + +- 1. Please refer to [FastDeploy Service-based Deployment](../../../../../serving/README_CN.md) for the hardware and software environment requirements of the service image and the image pull command + + +## Start the service + +```bash +#Download deployment sample code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/perception/paddle3d/smoke/serving + +#Download the Smoke model file and test image +wget https://bj.bcebos.com/fastdeploy/models/smoke.tar.gz +tar -xf smoke.tar.gz +wget https://bj.bcebos.com/fastdeploy/models/smoke_test.png + +# Put the configuration file into the preprocessing directory +mv smoke/infer_cfg.yml models/preprocess/1/ + +# Put the model into the models/runtime/1 directory, and rename it to model.pdmodel and model.pdiparams +mv smoke/smoke.pdmodel models/runtime/1/model.pdmodel +mv smoke/smoke.pdiparams models/runtime/1/model.pdiparams + +# Pull the fastdeploy image (x.y.z is the image version number, which needs to be replaced with the fastdeploy version number) +# GPU mirroring +docker pull registry.baidubce.com/paddlepaddle/fastdeploy:x.y.z-gpu-cuda11.4-trt8.4-21.10 +# CPU mirroring +docker pull paddlepaddle/fastdeploy:z.y.z-cpu-only-21.10 + + +# Run the container. The container name is fd_serving, and mount the current directory as the /serving directory of the container +nvidia-docker run -it --net=host --name fd_serving --shm-size="1g" -v `pwd`/:/serving registry.baidubce.com/paddlepaddle/fastdeploy:x.y.z-gpu-cuda11.4 -trt8.4-21.10 bash + +# Start the service (if you do not set the CUDA_VISIBLE_DEVICES environment variable, you will have the scheduling authority of all GPU cards) +CUDA_VISIBLE_DEVICES=0 fastdeployserver --model-repository=/serving/models +``` +>> **NOTE**: + +>> For pulling images, please refer to [Serving Deployment Main Document](../../../../../serving/README_CN.md) + +>> Execute fastdeployserver to start the service and "Address already in use" appears, please use `--grpc-port` to specify the grpc port number to start the service, and change the request port number in the client example. + +>> Other startup parameters can be viewed using fastdeployserver --help + +After the service starts successfully, there will be the following output: +``` + … +I0928 04:51:15.784517 206 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I0928 04:51:15.785177 206 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I0928 04:51:15.826578 206 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 +``` + + +## Client request + +Execute the following command on the physical machine, send the grpc request and output the result +``` +#Download test image +wget https://bj.bcebos.com/fastdeploy/models/smoke_test.png + +# Install client dependencies +python3 -m pip install tritonclient[all] + +# send request +python3 smoke_grpc_client.py +``` + +After sending the request successfully, the detection result in json format will be returned and printed out: +``` +output_name: PERCEPTION_RESULT +, 0.0068892366252839565] +label_ids: [2, 0, 1, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 , 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0] +``` + +## Configuration modification + +The current default configuration runs the Paddle engine on the CPU, if you want to run it on the GPU or other inference engines. It is necessary to modify the configuration in `models/runtime/config.pbtxt`, for details, please refer to [configuration document](../../../../../serving/docs/zh_CN/model_configuration.md) diff --git a/examples/vision/perception/paddle3d/smoke/serving/README_CN.md b/examples/vision/perception/paddle3d/smoke/serving/README_CN.md new file mode 100755 index 0000000000..e078d1a86c --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/README_CN.md @@ -0,0 +1,85 @@ +[English](README.md) | 简体中文 +# Smoke 服务化部署示例 + +本文档介绍 Paddle3D 中 Smoke 模型的服务化部署。 + +Smoke 模型导出和预训练模型下载请看[Smoke模型部署](../README.md)文档。 + +在服务化部署前，需确认 + +- 1. 服务化镜像的软硬件环境要求和镜像拉取命令请参考[FastDeploy服务化部署](../../../../../serving/README_CN.md) + + +## 启动服务 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/perception/paddle3d/smoke/serving + +#下载 Smoke 模型文件和测试图片 +wget https://bj.bcebos.com/fastdeploy/models/smoke.tar.gz +tar -xf smoke.tar.gz +wget https://bj.bcebos.com/fastdeploy/models/smoke_test.png + +# 将配置文件放入预处理目录 +mv smoke/infer_cfg.yml models/preprocess/1/ + +# 将模型放入 models/runtime/1 目录下, 并重命名为 model.pdmodel 和 model.pdiparams +mv smoke/smoke.pdmodel models/runtime/1/model.pdmodel +mv smoke/smoke.pdiparams models/runtime/1/model.pdiparams + +# 拉取 fastdeploy 镜像(x.y.z 为镜像版本号，需替换成 fastdeploy 版本数字) +# GPU 镜像 +docker pull registry.baidubce.com/paddlepaddle/fastdeploy:x.y.z-gpu-cuda11.4-trt8.4-21.10 +# CPU 镜像 +docker pull paddlepaddle/fastdeploy:z.y.z-cpu-only-21.10 + + +# 运行容器.容器名字为 fd_serving, 并挂载当前目录为容器的 /serving 目录 +nvidia-docker run -it --net=host --name fd_serving --shm-size="1g" -v `pwd`/:/serving registry.baidubce.com/paddlepaddle/fastdeploy:x.y.z-gpu-cuda11.4-trt8.4-21.10 bash + +# 启动服务(不设置 CUDA_VISIBLE_DEVICES 环境变量，会拥有所有 GPU 卡的调度权限) +CUDA_VISIBLE_DEVICES=0 fastdeployserver --model-repository=/serving/models +``` +>> **注意**: + +>> 拉取镜像请看[服务化部署主文档](../../../../../serving/README_CN.md) + +>> 执行 fastdeployserver 启动服务出现 "Address already in use", 请使用 `--grpc-port` 指定 grpc 端口号来启动服务，同时更改客户端示例中的请求端口号. + +>> 其他启动参数可以使用 fastdeployserver --help 查看 + +服务启动成功后， 会有以下输出: +``` +...... +I0928 04:51:15.784517 206 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I0928 04:51:15.785177 206 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I0928 04:51:15.826578 206 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 +``` + + +## 客户端请求 + +在物理机器中执行以下命令，发送 grpc 请求并输出结果 +``` +#下载测试图片 +wget https://bj.bcebos.com/fastdeploy/models/smoke_test.png + +#安装客户端依赖 +python3 -m pip install tritonclient[all] + +# 发送请求 +python3 smoke_grpc_client.py +``` + +发送请求成功后，会返回 json 格式的检测结果并打印输出: +``` +output_name: PERCEPTION_RESULT +scores: [0.8080164790153503, 0.03356542810797691, 0.03165825456380844, 0.020817330107092857, 0.018075695261359215, 0.017861749976873398, 0.016441335901618004, 0.01476177480071783, 0.012927377596497536, 0.012407636269927025, 0.012400650419294834, 0.012216777540743351, 0.01208423636853695, 0.011721019633114338, 0.011697308160364628, 0.011695655062794685, 0.011603309772908688, 0.011140472255647182, 0.010927721858024597, 0.01036786288022995, 0.00984608568251133, 0.009827949106693268, 0.009761993773281574, 0.00959752406924963, 0.009595031850039959, 0.009423951618373394, 0.008946355432271957, 0.008635037578642368, 0.008597995154559612, 0.008552121929824352, 0.00839947909116745, 0.008325068280100822, 0.00830004084855318, 0.00826205126941204, 0.008174785412847996, 0.008085251785814762, 0.008026468567550182, 0.00796759407967329, 0.007873599417507648, 0.007816540077328682, 0.007742374204099178, 0.007734378334134817, 0.0077047450467944145, 0.007684454321861267, 0.007525254040956497, 0.007521109655499458, 0.007519087754189968, 0.007399206515401602, 0.0071790567599236965, 0.0068892366252839565] +label_ids: [2, 0, 1, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +``` + +## 配置修改 + +当前默认配置在 CPU 上运行 Paddle 引擎， 如果要在 GPU 或其他推理引擎上运行。 需要修改 `models/runtime/config.pbtxt` 中配置，详情请参考[配置文档](../../../../../serving/docs/zh_CN/model_configuration.md) diff --git a/examples/vision/perception/paddle3d/smoke/serving/models/postprocess/1/model.py b/examples/vision/perception/paddle3d/smoke/serving/models/postprocess/1/model.py new file mode 100755 index 0000000000..6eda2a786d --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/models/postprocess/1/model.py @@ -0,0 +1,110 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import numpy as np +import time + +import fastdeploy as fd + +# triton_python_backend_utils is available in every Triton Python model. You +# need to use this module to create inference requests and responses. It also +# contains some utility functions for extracting information from model_config +# and converting Triton input/output types to numpy types. +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + """Your Python model must use the same class name. Every Python model + that is created must have "TritonPythonModel" as the class name. + """ + + def initialize(self, args): + """`initialize` is called only once when the model is being loaded. + Implementing `initialize` function is optional. This function allows + the model to intialize any state associated with this model. + Parameters + ---------- + args : dict + Both keys and values are strings. The dictionary keys and values are: + * model_config: A JSON string containing the model configuration + * model_instance_kind: A string containing model instance kind + * model_instance_device_id: A string containing model instance device ID + * model_repository: Model repository path + * model_version: Model version + * model_name: Model name + """ + # You must parse model_config. JSON string is not parsed here + self.model_config = json.loads(args['model_config']) + print("model_config:", self.model_config) + + self.input_names = [] + for input_config in self.model_config["input"]: + self.input_names.append(input_config["name"]) + print("postprocess input names:", self.input_names) + + self.output_names = [] + self.output_dtype = [] + for output_config in self.model_config["output"]: + self.output_names.append(output_config["name"]) + dtype = pb_utils.triton_string_to_numpy(output_config["data_type"]) + self.output_dtype.append(dtype) + print("postprocess output names:", self.output_names) + + self.postprocess_ = fd.vision.perception.SmokePostprocessor() + + def execute(self, requests): + """`execute` must be implemented in every Python model. `execute` + function receives a list of pb_utils.InferenceRequest as the only + argument. This function is called when an inference is requested + for this model. Depending on the batching configuration (e.g. Dynamic + Batching) used, `requests` may contain multiple requests. Every + Python model, must create one pb_utils.InferenceResponse for every + pb_utils.InferenceRequest in `requests`. If there is an error, you can + set the error argument when creating a pb_utils.InferenceResponse. + Parameters + ---------- + requests : list + A list of pb_utils.InferenceRequest + Returns + ------- + list + A list of pb_utils.InferenceResponse. The length of this list must + be the same as `requests` + """ + responses = [] + for request in requests: + infer_outputs = [] + for name in self.input_names: + infer_output = pb_utils.get_input_tensor_by_name(request, name) + if infer_output: + infer_output = infer_output.as_numpy() + infer_outputs.append(infer_output) + + results = self.postprocess_.run(infer_outputs) + r_str = fd.vision.utils.fd_result_to_json(results) + + r_np = np.array(r_str, dtype=np.object_) + out_tensor = pb_utils.Tensor(self.output_names[0], r_np) + inference_response = pb_utils.InferenceResponse( + output_tensors=[out_tensor, ]) + responses.append(inference_response) + return responses + + def finalize(self): + """`finalize` is called only once when the model is being unloaded. + Implementing `finalize` function is optional. This function allows + the model to perform any necessary clean ups before exit. + """ + print('Cleaning up...') diff --git a/examples/vision/perception/paddle3d/smoke/serving/models/postprocess/config.pbtxt b/examples/vision/perception/paddle3d/smoke/serving/models/postprocess/config.pbtxt new file mode 100755 index 0000000000..e1ab650a1c --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/models/postprocess/config.pbtxt @@ -0,0 +1,25 @@ +name: "postprocess" +backend: "python" + +input [ + { + name: "post_input1" + data_type: TYPE_FP32 + dims: [ 50, 14 ] + } +] + +output [ + { + name: "post_output" + data_type: TYPE_STRING + dims: [ -1 ] + } +] + +instance_group [ + { + count: 1 + kind: KIND_CPU + } +] \ No newline at end of file diff --git a/examples/vision/perception/paddle3d/smoke/serving/models/preprocess/1/model.py b/examples/vision/perception/paddle3d/smoke/serving/models/preprocess/1/model.py new file mode 100755 index 0000000000..f320115a6b --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/models/preprocess/1/model.py @@ -0,0 +1,114 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import numpy as np +import os + +import fastdeploy as fd + +# triton_python_backend_utils is available in every Triton Python model. You +# need to use this module to create inference requests and responses. It also +# contains some utility functions for extracting information from model_config +# and converting Triton input/output types to numpy types. +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + """Your Python model must use the same class name. Every Python model + that is created must have "TritonPythonModel" as the class name. + """ + + def initialize(self, args): + """`initialize` is called only once when the model is being loaded. + Implementing `initialize` function is optional. This function allows + the model to intialize any state associated with this model. + Parameters + ---------- + args : dict + Both keys and values are strings. The dictionary keys and values are: + * model_config: A JSON string containing the model configuration + * model_instance_kind: A string containing model instance kind + * model_instance_device_id: A string containing model instance device ID + * model_repository: Model repository path + * model_version: Model version + * model_name: Model name + """ + # You must parse model_config. JSON string is not parsed here + self.model_config = json.loads(args['model_config']) + print("model_config:", self.model_config) + + self.input_names = [] + for input_config in self.model_config["input"]: + self.input_names.append(input_config["name"]) + print("preprocess input names:", self.input_names) + + self.output_names = [] + self.output_dtype = [] + for output_config in self.model_config["output"]: + self.output_names.append(output_config["name"]) + # dtype = pb_utils.triton_string_to_numpy(output_config["data_type"]) + # self.output_dtype.append(dtype) + self.output_dtype.append(output_config["data_type"]) + print("preprocess output names:", self.output_names) + + # init PaddleClasPreprocess class + yaml_path = os.path.abspath(os.path.dirname( + __file__)) + "/infer_cfg.yml" + self.preprocess_ = fd.vision.perception.SmokePreprocessor(yaml_path) + + def execute(self, requests): + """`execute` must be implemented in every Python model. `execute` + function receives a list of pb_utils.InferenceRequest as the only + argument. This function is called when an inference is requested + for this model. Depending on the batching configuration (e.g. Dynamic + Batching) used, `requests` may contain multiple requests. Every + Python model, must create one pb_utils.InferenceResponse for every + pb_utils.InferenceRequest in `requests`. If there is an error, you can + set the error argument when creating a pb_utils.InferenceResponse. + Parameters + ---------- + requests : list + A list of pb_utils.InferenceRequest + Returns + ------- + list + A list of pb_utils.InferenceResponse. The length of this list must + be the same as `requests` + """ + responses = [] + for request in requests: + data = pb_utils.get_input_tensor_by_name(request, + self.input_names[0]) + data = data.as_numpy() + outputs = self.preprocess_.run(data) + + output_tensors = [] + for idx, name in enumerate(self.output_names): + dlpack_tensor = outputs[idx].to_dlpack() + output_tensor = pb_utils.Tensor.from_dlpack(name, + dlpack_tensor) + output_tensors.append(output_tensor) + + inference_response = pb_utils.InferenceResponse( + output_tensors=output_tensors) + responses.append(inference_response) + return responses + + def finalize(self): + """`finalize` is called only once when the model is being unloaded. + Implementing `finalize` function is optional. This function allows + the model to perform any necessary clean ups before exit. + """ + print('Cleaning up...') diff --git a/examples/vision/perception/paddle3d/smoke/serving/models/preprocess/config.pbtxt b/examples/vision/perception/paddle3d/smoke/serving/models/preprocess/config.pbtxt new file mode 100755 index 0000000000..b49df85b59 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/models/preprocess/config.pbtxt @@ -0,0 +1,35 @@ +name: "preprocess" +backend: "python" + +input [ + { + name: "preprocess_input" + data_type: TYPE_UINT8 + dims: [ 1, -1, -1, 3 ] + } +] + +output [ + { + name: "preprocess_output1" + data_type: TYPE_FP32 + dims: [ 1, 2 ] + }, + { + name: "preprocess_output2" + data_type: TYPE_FP32 + dims: [ 1, 3, -1, -1 ] + }, + { + name: "preprocess_output3" + data_type: TYPE_FP32 + dims: [ 1, 3, 3 ] + } +] + +instance_group [ + { + count: 1 + kind: KIND_CPU + } +] \ No newline at end of file diff --git a/examples/vision/perception/paddle3d/smoke/serving/models/runtime/1/README.md b/examples/vision/perception/paddle3d/smoke/serving/models/runtime/1/README.md new file mode 100755 index 0000000000..1e5d914b43 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/models/runtime/1/README.md @@ -0,0 +1,5 @@ +# Runtime Directory + +This directory holds the model files. +Paddle models must be model.pdmodel and model.pdiparams files. +ONNX models must be model.onnx files. diff --git a/examples/vision/perception/paddle3d/smoke/serving/models/runtime/config.pbtxt b/examples/vision/perception/paddle3d/smoke/serving/models/runtime/config.pbtxt new file mode 100755 index 0000000000..461750c26d --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/models/runtime/config.pbtxt @@ -0,0 +1,57 @@ +# optional, If name is specified it must match the name of the model repository directory containing the model. +name: "runtime" +backend: "fastdeploy" + +# Input configuration of the model +input [ + { + # input name + name: "down_ratios" + # input type such as TYPE_FP32、TYPE_UINT8、TYPE_INT8、TYPE_INT16、TYPE_INT32、TYPE_INT64、TYPE_FP16、TYPE_STRING + data_type: TYPE_FP32 + dims: [ 1, 2 ] + }, + { + # input name + name: "images" + # input type such as TYPE_FP32、TYPE_UINT8、TYPE_INT8、TYPE_INT16、TYPE_INT32、TYPE_INT64、TYPE_FP16、TYPE_STRING + data_type: TYPE_FP32 + dims: [ 1, 3, -1, -1 ] + }, + { + name: "trans_cam_to_img" + data_type: TYPE_FP32 + dims: [ 1, 3, 3 ] + } +] + +# The output of the model is configured in the same format as the input +output [ + { + name: "concat_13.tmp_0" + data_type: TYPE_FP32 + dims: [ 50, 14 ] + } +] + +# Number of instances of the model +instance_group [ + { + # The number of instances is 1 + count: 1 + # Use GPU, CPU inference option is:KIND_CPU + kind: KIND_CPU + # The instance is deployed on the 0th GPU card + # gpus: [0] + } +] + +optimization { + execution_accelerators { + cpu_execution_accelerator : [ { + # use Paddle engine + name: "paddle", + parameters { key: "use_mkldnn" value: "1" } + } + ] +}} diff --git a/examples/vision/perception/paddle3d/smoke/serving/models/smoke/1/README.md b/examples/vision/perception/paddle3d/smoke/serving/models/smoke/1/README.md new file mode 100755 index 0000000000..53ae122872 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/models/smoke/1/README.md @@ -0,0 +1,3 @@ +# Smoke Pipeline + +The pipeline directory does not have model files, but a version number directory needs to be maintained. diff --git a/examples/vision/perception/paddle3d/smoke/serving/models/smoke/config.pbtxt b/examples/vision/perception/paddle3d/smoke/serving/models/smoke/config.pbtxt new file mode 100755 index 0000000000..3500b6f3e8 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/models/smoke/config.pbtxt @@ -0,0 +1,72 @@ +platform: "ensemble" + +input [ + { + name: "INPUT" + data_type: TYPE_UINT8 + dims: [ 1, -1, -1, 3 ] + } +] +output [ + { + name: "PERCEPTION_RESULT" + data_type: TYPE_STRING + dims: [ -1 ] + } +] +ensemble_scheduling { + step [ + { + model_name: "preprocess" + model_version: 1 + input_map { + key: "preprocess_input" + value: "INPUT" + } + output_map { + key: "preprocess_output1" + value: "RUNTIME_INPUT1" + } + output_map { + key: "preprocess_output2" + value: "RUNTIME_INPUT2" + } + output_map { + key: "preprocess_output3" + value: "RUNTIME_INPUT3" + } + }, + { + model_name: "runtime" + model_version: 1 + input_map { + key: "down_ratios" + value: "RUNTIME_INPUT1" + } + input_map { + key: "images" + value: "RUNTIME_INPUT2" + } + input_map { + key: "trans_cam_to_img" + value: "RUNTIME_INPUT3" + } + output_map { + key: "concat_13.tmp_0" + value: "RUNTIME_OUTPUT1" + } + }, + { + model_name: "postprocess" + model_version: 1 + input_map { + key: "post_input1" + value: "RUNTIME_OUTPUT1" + } + output_map { + key: "post_output" + value: "PERCEPTION_RESULT" + } + } + ] +} \ No newline at end of file diff --git a/examples/vision/perception/paddle3d/smoke/serving/smoke_grpc_client.py b/examples/vision/perception/paddle3d/smoke/serving/smoke_grpc_client.py new file mode 100755 index 0000000000..5bdd973ee0 --- /dev/null +++ b/examples/vision/perception/paddle3d/smoke/serving/smoke_grpc_client.py @@ -0,0 +1,108 @@ +import logging +import numpy as np +import time +from typing import Optional +import cv2 +import json + +from tritonclient import utils as client_utils +from tritonclient.grpc import InferenceServerClient, InferInput, InferRequestedOutput, service_pb2_grpc, service_pb2 + +LOGGER = logging.getLogger("run_inference_on_triton") + + +class SyncGRPCTritonRunner: + DEFAULT_MAX_RESP_WAIT_S = 120 + + def __init__( + self, + server_url: str, + model_name: str, + model_version: str, + *, + verbose=False, + resp_wait_s: Optional[float]=None, ): + self._server_url = server_url + self._model_name = model_name + self._model_version = model_version + self._verbose = verbose + self._response_wait_t = self.DEFAULT_MAX_RESP_WAIT_S if resp_wait_s is None else resp_wait_s + + self._client = InferenceServerClient( + self._server_url, verbose=self._verbose) + error = self._verify_triton_state(self._client) + if error: + raise RuntimeError( + f"Could not communicate to Triton Server: {error}") + + LOGGER.debug( + f"Triton server {self._server_url} and model {self._model_name}:{self._model_version} " + f"are up and ready!") + + model_config = self._client.get_model_config(self._model_name, + self._model_version) + model_metadata = self._client.get_model_metadata(self._model_name, + self._model_version) + LOGGER.info(f"Model config {model_config}") + LOGGER.info(f"Model metadata {model_metadata}") + + for tm in model_metadata.inputs: + print("tm:", tm) + self._inputs = {tm.name: tm for tm in model_metadata.inputs} + self._input_names = list(self._inputs) + self._outputs = {tm.name: tm for tm in model_metadata.outputs} + self._output_names = list(self._outputs) + self._outputs_req = [ + InferRequestedOutput(name) for name in self._outputs + ] + + def Run(self, inputs): + """ + Args: + inputs: list, Each value corresponds to an input name of self._input_names + Returns: + results: dict, {name : numpy.array} + """ + infer_inputs = [] + for idx, data in enumerate(inputs): + infer_input = InferInput(self._input_names[idx], data.shape, + "UINT8") + infer_input.set_data_from_numpy(data) + infer_inputs.append(infer_input) + + results = self._client.infer( + model_name=self._model_name, + model_version=self._model_version, + inputs=infer_inputs, + outputs=self._outputs_req, + client_timeout=self._response_wait_t, ) + results = {name: results.as_numpy(name) for name in self._output_names} + return results + + def _verify_triton_state(self, triton_client): + if not triton_client.is_server_live(): + return f"Triton server {self._server_url} is not live" + elif not triton_client.is_server_ready(): + return f"Triton server {self._server_url} is not ready" + elif not triton_client.is_model_ready(self._model_name, + self._model_version): + return f"Model {self._model_name}:{self._model_version} is not ready" + return None + + +if __name__ == "__main__": + model_name = "smoke" + model_version = "1" + url = "localhost:8001" + runner = SyncGRPCTritonRunner(url, model_name, model_version) + im = cv2.imread("smoke_test.png") + im = np.array([im, ]) + for i in range(1): + result = runner.Run([im, ]) + for name, values in result.items(): + print("output_name:", name) + # values is batch + for value in values: + value = json.loads(value) + print("scores: ", value['scores']) + print("label_ids: ", value['label_ids']) diff --git a/fastdeploy/runtime/backends/paddle/paddle_backend.cc b/fastdeploy/runtime/backends/paddle/paddle_backend.cc old mode 100755 new mode 100644 index 562b1ce734..99e2ab117f --- a/fastdeploy/runtime/backends/paddle/paddle_backend.cc +++ b/fastdeploy/runtime/backends/paddle/paddle_backend.cc @@ -226,10 +226,17 @@ bool PaddleBackend::InitFromPaddle(const std::string& model, std::map> min_shape; std::map> opt_shape; GetDynamicShapeFromOption(option, &max_shape, &min_shape, &opt_shape); + std::map> max_input_data; + std::map> min_input_data; + std::map> opt_input_data; + if (!option.trt_option.min_input_data.empty()) { + GetInputDataFromOption(option, &max_input_data, &min_input_data, + &opt_input_data); + } // Need to run once to get the shape range info file. - CollectShapeRun(predictor_tmp.get(), max_shape); - CollectShapeRun(predictor_tmp.get(), min_shape); - CollectShapeRun(predictor_tmp.get(), opt_shape); + CollectShapeRun(predictor_tmp.get(), max_shape, max_input_data); + CollectShapeRun(predictor_tmp.get(), min_shape, min_input_data); + CollectShapeRun(predictor_tmp.get(), opt_shape, min_input_data); FDINFO << "Finish generating shape range info file." << std::endl; } FDINFO << "Start loading shape range info file " << shape_range_info @@ -400,9 +407,33 @@ void PaddleBackend::GetDynamicShapeFromOption( } } +void PaddleBackend::GetInputDataFromOption( + const PaddleBackendOption& option, + std::map>* max_input_data, + std::map>* min_input_data, + std::map>* opt_input_data) const { + for (const auto& item : option.trt_option.min_input_data) { + auto max_iter = option.trt_option.max_input_data.find(item.first); + auto opt_iter = option.trt_option.opt_input_data.find(item.first); + FDASSERT(max_iter != option.trt_option.max_input_data.end(), + "Cannot find %s in TrtBackendOption::min_input_data.", + item.first.c_str()); + FDASSERT(opt_iter != option.trt_option.opt_input_data.end(), + "Cannot find %s in TrtBackendOption::opt_input_data.", + item.first.c_str()); + (*max_input_data)[item.first].assign(max_iter->second.begin(), + max_iter->second.end()); + (*opt_input_data)[item.first].assign(opt_iter->second.begin(), + opt_iter->second.end()); + (*min_input_data)[item.first].assign(item.second.begin(), + item.second.end()); + } +} + void PaddleBackend::CollectShapeRun( paddle_infer::Predictor* predictor, - const std::map>& shape) const { + const std::map>& shape, + const std::map>& data) const { auto input_names = predictor->GetInputNames(); auto input_type = predictor->GetInputTypes(); for (const auto& name : input_names) { @@ -418,21 +449,47 @@ void PaddleBackend::CollectShapeRun( int shape_num = std::accumulate(shape_value.begin(), shape_value.end(), 1, std::multiplies()); tensor->Reshape(shape_value); + + if (data.find(name) != data.end()) { + FDASSERT(data.at(name).size() == shape_num, + "The data num and accumulate of shape must be equal for input: " + "[\"%s\"], " + " When Use the (C++)RuntimeOption.trt_option.SetInputData/ " + " (Python)RuntimeOption.trt_option.set_input_data/", + name.c_str()); + } + auto dtype = input_type[name]; switch (dtype) { case paddle_infer::DataType::FLOAT32: { - std::vector input_data(shape_num, 1.0); - tensor->CopyFromCpu(input_data.data()); + if (data.find(name) != data.end()) { + tensor->CopyFromCpu(data.at(name).data()); + } else { + std::vector input_data(shape_num, 1.0); + tensor->CopyFromCpu(input_data.data()); + } break; } case paddle_infer::DataType::INT32: { - std::vector input_data(shape_num, 1); - tensor->CopyFromCpu(input_data.data()); + if (data.find(name) != data.end()) { + std::vector input_data(data.at(name).begin(), + data.at(name).end()); + tensor->CopyFromCpu(input_data.data()); + } else { + std::vector input_data(shape_num, 1); + tensor->CopyFromCpu(input_data.data()); + } break; } case paddle_infer::DataType::INT64: { - std::vector input_data(shape_num, 1); - tensor->CopyFromCpu(input_data.data()); + if (data.find(name) != data.end()) { + std::vector input_data(data.at(name).begin(), + data.at(name).end()); + tensor->CopyFromCpu(input_data.data()); + } else { + std::vector input_data(shape_num, 1); + tensor->CopyFromCpu(input_data.data()); + } break; } default: { diff --git a/fastdeploy/runtime/backends/paddle/paddle_backend.h b/fastdeploy/runtime/backends/paddle/paddle_backend.h index b1b0dbd4c6..7ca8c0b14c 100755 --- a/fastdeploy/runtime/backends/paddle/paddle_backend.h +++ b/fastdeploy/runtime/backends/paddle/paddle_backend.h @@ -81,12 +81,18 @@ class PaddleBackend : public BaseBackend { void CollectShapeRun(paddle_infer::Predictor* predictor, - const std::map>& shape) const; + const std::map>& shape, + const std::map>& data) const; void GetDynamicShapeFromOption( const PaddleBackendOption& option, std::map>* max_shape, std::map>* min_shape, std::map>* opt_shape) const; + void GetInputDataFromOption( + const PaddleBackendOption& option, + std::map>* max_input_data, + std::map>* min_input_data, + std::map>* opt_input_data) const; void SetTRTDynamicShapeToConfig(const PaddleBackendOption& option); PaddleBackendOption option_; paddle_infer::Config config_; diff --git a/fastdeploy/runtime/backends/tensorrt/option.h b/fastdeploy/runtime/backends/tensorrt/option.h index 48b8f83fbc..674af34574 100755 --- a/fastdeploy/runtime/backends/tensorrt/option.h +++ b/fastdeploy/runtime/backends/tensorrt/option.h @@ -33,8 +33,9 @@ struct TrtBackendOption { /// Enable log while converting onnx model to tensorrt bool enable_log_info = false; - - /// Enable half precison inference, on some device not support half precision, it will fallback to float32 mode + + /// Enable half precison inference, on some device not support half precision, + /// it will fallback to float32 mode bool enable_fp16 = false; /** \brief Set shape range of input tensor for the model that contain dynamic input shape while using TensorRT backend @@ -63,9 +64,44 @@ struct TrtBackendOption { max_shape[tensor_name].assign(max.begin(), max.end()); } } - /// Set cache file path while use TensorRT backend. Loadding a Paddle/ONNX model and initialize TensorRT will take a long time, by this interface it will save the tensorrt engine to `cache_file_path`, and load it directly while execute the code again + + /** \brief Set Input data for input tensor for the model while using TensorRT backend + * + * \param[in] tensor_name The name of input for the model which is dynamic shape + * \param[in] min_data The input data for minimal shape for the input tensor + * \param[in] opt_data The input data for optimized shape for the input tensor + * \param[in] max_data The input data for maximum shape for the input tensor, if set as default value, it will keep same with min_data + */ + void SetInputData(const std::string& tensor_name, + const std::vector min_data, + const std::vector opt_data = std::vector(), + const std::vector max_data = std::vector()) { + max_input_data[tensor_name].clear(); + min_input_data[tensor_name].clear(); + opt_input_data[tensor_name].clear(); + min_input_data[tensor_name].assign(min_data.begin(), min_data.end()); + if (opt_data.empty()) { + opt_input_data[tensor_name].assign(min_data.begin(), min_data.end()); + } else { + opt_input_data[tensor_name].assign(opt_data.begin(), opt_data.end()); + } + if (max_data.empty()) { + max_input_data[tensor_name].assign(min_data.begin(), min_data.end()); + } else { + max_input_data[tensor_name].assign(max_data.begin(), max_data.end()); + } + } + + /// Set cache file path while use TensorRT backend. + /// Loadding a Paddle/ONNX model and initialize TensorRT will + /// take a long time, + /// by this interface it will save the tensorrt engine to `cache_file_path`, + /// and load it directly while execute the code again std::string serialize_file = ""; + std::map> max_input_data; + std::map> min_input_data; + std::map> opt_input_data; // The below parameters may be removed in next version, please do not // visit or use them directly std::map> max_shape; diff --git a/fastdeploy/runtime/backends/tensorrt/option_pybind.cc b/fastdeploy/runtime/backends/tensorrt/option_pybind.cc old mode 100644 new mode 100755 index f46f27f95a..b9aa8fa20b --- a/fastdeploy/runtime/backends/tensorrt/option_pybind.cc +++ b/fastdeploy/runtime/backends/tensorrt/option_pybind.cc @@ -26,7 +26,8 @@ void BindTrtOption(pybind11::module& m) { .def_readwrite("max_workspace_size", &TrtBackendOption::max_workspace_size) .def_readwrite("serialize_file", &TrtBackendOption::serialize_file) - .def("set_shape", &TrtBackendOption::SetShape); + .def("set_shape", &TrtBackendOption::SetShape) + .def("set_input_data", &TrtBackendOption::SetInputData); } } // namespace fastdeploy diff --git a/fastdeploy/runtime/runtime_option.cc b/fastdeploy/runtime/runtime_option.cc index 8d18637a86..b5d0e85dfe 100644 --- a/fastdeploy/runtime/runtime_option.cc +++ b/fastdeploy/runtime/runtime_option.cc @@ -379,6 +379,17 @@ void RuntimeOption::SetTrtInputShape(const std::string& input_name, trt_option.SetShape(input_name, min_shape, opt_shape, max_shape); } +void RuntimeOption::SetTrtInputData(const std::string& input_name, + const std::vector& min_shape_data, + const std::vector& opt_shape_data, + const std::vector& max_shape_data) { + FDWARNING << "`RuntimeOption::SetTrtInputData` will be removed in v1.2.0, " + "please use `RuntimeOption.trt_option.SetInputData()` instead." + << std::endl; + trt_option.SetInputData(input_name, min_shape_data, opt_shape_data, + max_shape_data); +} + void RuntimeOption::SetTrtMaxWorkspaceSize(size_t max_workspace_size) { FDWARNING << "`RuntimeOption::SetTrtMaxWorkspaceSize` will be removed in " "v1.2.0, please modify its member variable directly, e.g " diff --git a/fastdeploy/runtime/runtime_option.h b/fastdeploy/runtime/runtime_option.h index 51032bae18..8c3f82e63d 100755 --- a/fastdeploy/runtime/runtime_option.h +++ b/fastdeploy/runtime/runtime_option.h @@ -257,6 +257,12 @@ struct FASTDEPLOY_DECL RuntimeOption { const std::string& input_name, const std::vector& min_shape, const std::vector& opt_shape = std::vector(), const std::vector& max_shape = std::vector()); + + void SetTrtInputData( + const std::string& input_name, const std::vector& min_shape_data, + const std::vector& opt_shape_data = std::vector(), + const std::vector& max_shape_data = std::vector()); + void SetTrtMaxWorkspaceSize(size_t trt_max_workspace_size); void SetTrtMaxBatchSize(size_t max_batch_size); void EnableTrtFP16(); diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h old mode 100644 new mode 100755 index 9051c66fc4..00dfa7ffab --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -32,6 +32,7 @@ #include "fastdeploy/vision/detection/contrib/yolov8/yolov8.h" #include "fastdeploy/vision/detection/contrib/yolox.h" #include "fastdeploy/vision/detection/contrib/rknpu2/model.h" +#include "fastdeploy/vision/perception/paddle3d/smoke/smoke.h" #include "fastdeploy/vision/detection/ppdet/model.h" #include "fastdeploy/vision/facealign/contrib/face_landmark_1000.h" #include "fastdeploy/vision/facealign/contrib/pfld.h" diff --git a/fastdeploy/vision/common/result.cc b/fastdeploy/vision/common/result.cc index 1c8d20c0fc..262eeb83a3 100755 --- a/fastdeploy/vision/common/result.cc +++ b/fastdeploy/vision/common/result.cc @@ -166,6 +166,96 @@ std::string DetectionResult::Str() { return out; } +// PerceptionResult ----------------------------------------------------- +PerceptionResult::PerceptionResult(const PerceptionResult& res) { + scores.assign(res.scores.begin(), res.scores.end()); + label_ids.assign(res.label_ids.begin(), res.label_ids.end()); + boxes.assign(res.boxes.begin(), res.boxes.end()); + center.assign(res.center.begin(), res.center.end()); + observation_angle.assign(res.observation_angle.begin(), + res.observation_angle.end()); + yaw_angle.assign(res.yaw_angle.begin(), res.yaw_angle.end()); + velocity.assign(res.velocity.begin(), res.velocity.end()); +} + +PerceptionResult& PerceptionResult::operator=(PerceptionResult&& other) { + if (&other != this) { + scores = std::move(other.scores); + label_ids = std::move(other.label_ids); + boxes = std::move(other.boxes); + center = std::move(other.center); + observation_angle = std::move(other.observation_angle); + yaw_angle = std::move(other.yaw_angle); + velocity = std::move(other.velocity); + } + return *this; +} + +void PerceptionResult::Free() { + std::vector().swap(scores); + std::vector().swap(label_ids); + std::vector>().swap(boxes); + std::vector>().swap(center); + std::vector().swap(observation_angle); + std::vector().swap(yaw_angle); + std::vector>().swap(velocity); +} + +void PerceptionResult::Clear() { + scores.clear(); + label_ids.clear(); + boxes.clear(); + center.clear(); + observation_angle.clear(); + yaw_angle.clear(); + velocity.clear(); +} + +void PerceptionResult::Reserve(int size) { + scores.reserve(size); + label_ids.reserve(size); + boxes.reserve(size); + center.reserve(size); + observation_angle.reserve(size); + yaw_angle.reserve(size); + velocity.reserve(size); +} + +void PerceptionResult::Resize(int size) { + scores.resize(size); + label_ids.resize(size); + boxes.resize(size); + center.resize(size); + observation_angle.resize(size); + yaw_angle.resize(size); + velocity.resize(size); +} + +std::string PerceptionResult::Str() { + std::string out; + out = + "PerceptionResult: [xmin, ymin, xmax, ymax, w, h, l, cx, cy, cz, " + "yaw_angle, " + "ob_angle, score, label_id]\n"; + for (size_t i = 0; i < boxes.size(); ++i) { + out = out + std::to_string(boxes[i][0]) + "," + + std::to_string(boxes[i][1]) + ", " + std::to_string(boxes[i][2]) + + ", " + std::to_string(boxes[i][3]) + ", " + + std::to_string(boxes[i][4]) + ", " + std::to_string(boxes[i][5]) + + ", " + std::to_string(boxes[i][6]) + ", " + + std::to_string(center[i][0]) + ", " + std::to_string(center[i][1]) + + ", " + std::to_string(center[i][2]) + ", " + + std::to_string(yaw_angle[i]) + ", " + + std::to_string(observation_angle[i]) + ", " + + std::to_string(scores[i]) + ", " + std::to_string(label_ids[i]); + + out += "\n"; + } + return out; +} + +// PerceptionResult finished + void KeyPointDetectionResult::Free() { std::vector>().swap(keypoints); std::vector().swap(scores); @@ -531,8 +621,8 @@ std::string OCRResult::Str() { out = out + "]"; if (rec_scores.size() > 0) { - out = out + "rec text: " + text[n] + " rec score:" + - std::to_string(rec_scores[n]) + " "; + out = out + "rec text: " + text[n] + + " rec score:" + std::to_string(rec_scores[n]) + " "; } if (cls_labels.size() > 0) { out = out + "cls label: " + std::to_string(cls_labels[n]) + @@ -546,8 +636,8 @@ std::string OCRResult::Str() { cls_scores.size() > 0) { std::string out; for (int i = 0; i < rec_scores.size(); i++) { - out = out + "rec text: " + text[i] + " rec score:" + - std::to_string(rec_scores[i]) + " "; + out = out + "rec text: " + text[i] + + " rec score:" + std::to_string(rec_scores[i]) + " "; out = out + "cls label: " + std::to_string(cls_labels[i]) + " cls score: " + std::to_string(cls_scores[i]); out = out + "\n"; @@ -566,8 +656,8 @@ std::string OCRResult::Str() { cls_scores.size() == 0) { std::string out; for (int i = 0; i < rec_scores.size(); i++) { - out = out + "rec text: " + text[i] + " rec score:" + - std::to_string(rec_scores[i]) + " "; + out = out + "rec text: " + text[i] + + " rec score:" + std::to_string(rec_scores[i]) + " "; out = out + "\n"; } return out; @@ -589,9 +679,9 @@ std::string HeadPoseResult::Str() { std::string out; out = "HeadPoseResult: [yaw, pitch, roll]\n"; - out = out + "yaw: " + std::to_string(euler_angles[0]) + "\n" + "pitch: " + - std::to_string(euler_angles[1]) + "\n" + "roll: " + - std::to_string(euler_angles[2]) + "\n"; + out = out + "yaw: " + std::to_string(euler_angles[0]) + "\n" + + "pitch: " + std::to_string(euler_angles[1]) + "\n" + + "roll: " + std::to_string(euler_angles[2]) + "\n"; return out; } diff --git a/fastdeploy/vision/common/result.h b/fastdeploy/vision/common/result.h index 6b40bf3144..4508e62af4 100755 --- a/fastdeploy/vision/common/result.h +++ b/fastdeploy/vision/common/result.h @@ -140,6 +140,44 @@ struct FASTDEPLOY_DECL DetectionResult : public BaseResult { std::string Str(); }; +/*! @brief Detection result structure for all the object detection models and instance segmentation models + */ +struct FASTDEPLOY_DECL PerceptionResult : public BaseResult { + PerceptionResult() = default; + + std::vector scores; + + std::vector label_ids; + // xmin, ymin, xmax, ymax, h, w, l + std::vector> boxes; + // cx, cy, cz + std::vector> center; + + std::vector observation_angle; + + std::vector yaw_angle; + // vx, vy, vz + std::vector> velocity; + + /// Copy constructor + PerceptionResult(const PerceptionResult& res); + /// Move assignment + PerceptionResult& operator=(PerceptionResult&& other); + + /// Clear PerceptionResult + void Clear(); + + /// Clear PerceptionResult and free the memory + void Free(); + + void Reserve(int size); + + void Resize(int size); + + /// Debug function, convert the result to string to print + std::string Str(); +}; + /*! @brief KeyPoint Detection result structure for all the keypoint detection models */ struct FASTDEPLOY_DECL KeyPointDetectionResult : public BaseResult { diff --git a/fastdeploy/vision/perception/paddle3d/smoke/postprocessor.cc b/fastdeploy/vision/perception/paddle3d/smoke/postprocessor.cc new file mode 100644 index 0000000000..4b2009d71b --- /dev/null +++ b/fastdeploy/vision/perception/paddle3d/smoke/postprocessor.cc @@ -0,0 +1,58 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/perception/paddle3d/smoke/postprocessor.h" + +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { +namespace vision { +namespace perception { + +SmokePostprocessor::SmokePostprocessor() {} + +bool SmokePostprocessor::Run(const std::vector& tensors, + std::vector* results) { + results->resize(1); + (*results)[0].Clear(); + (*results)[0].Reserve(tensors[0].shape[0]); + if (tensors[0].dtype != FDDataType::FP32) { + FDERROR << "Only support post process with float32 data." << std::endl; + return false; + } + const float* data = reinterpret_cast(tensors[0].Data()); + auto result = &(*results)[0]; + for (int i = 0; i < tensors[0].shape[0] * tensors[0].shape[1]; i += 14) { + // item 1 : class + // item 2 : observation angle α + // item 3 ~ 6 : box2d x1, y1, x2, y2 + // item 7 ~ 9 : box3d h, w, l + // item 10 ~ 12 : box3d bottom center x, y, z + // item 13 : box3d yaw angle + // item 14 : score + std::vector vec(data + i, data + i + 14); + result->scores.push_back(vec[13]); + result->label_ids.push_back(vec[0]); + result->boxes.emplace_back(std::array{ + vec[2], vec[3], vec[4], vec[5], vec[6], vec[7], vec[8]}); + result->center.emplace_back(std::array{vec[9], vec[10], vec[11]}); + result->observation_angle.push_back(vec[1]); + result->yaw_angle.push_back(vec[12]); + } + return true; +} + +} // namespace perception +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/perception/paddle3d/smoke/postprocessor.h b/fastdeploy/vision/perception/paddle3d/smoke/postprocessor.h new file mode 100755 index 0000000000..48135f61ea --- /dev/null +++ b/fastdeploy/vision/perception/paddle3d/smoke/postprocessor.h @@ -0,0 +1,48 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace perception { +/*! @brief Postprocessor object for Smoke serials model. + */ +class FASTDEPLOY_DECL SmokePostprocessor { + public: + /** \brief Create a postprocessor instance for Smoke serials model + */ + SmokePostprocessor(); + + /** \brief Process the result of runtime and fill to PerceptionResult structure + * + * \param[in] tensors The inference result from runtime + * \param[in] result The output result of detection + * \param[in] ims_info The shape info list, record input_shape and output_shape + * \return true if the postprocess successed, otherwise false + */ + bool Run(const std::vector& tensors, + std::vector* results); + + + protected: + float conf_threshold_; +}; + +} // namespace perception +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/perception/paddle3d/smoke/preprocessor.cc b/fastdeploy/vision/perception/paddle3d/smoke/preprocessor.cc new file mode 100755 index 0000000000..44c4af4b84 --- /dev/null +++ b/fastdeploy/vision/perception/paddle3d/smoke/preprocessor.cc @@ -0,0 +1,161 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/perception/paddle3d/smoke/preprocessor.h" + +#include "fastdeploy/function/concat.h" +#include "yaml-cpp/yaml.h" + +namespace fastdeploy { +namespace vision { +namespace perception { + +SmokePreprocessor::SmokePreprocessor(const std::string& config_file) { + config_file_ = config_file; + FDASSERT(BuildPreprocessPipelineFromConfig(), + "Failed to create Paddle3DDetPreprocessor."); + initialized_ = true; +} + +bool SmokePreprocessor::BuildPreprocessPipelineFromConfig() { + processors_.clear(); + YAML::Node cfg; + try { + cfg = YAML::LoadFile(config_file_); + } catch (YAML::BadFile& e) { + FDERROR << "Failed to load yaml file " << config_file_ + << ", maybe you should check this file." << std::endl; + return false; + } + + // read for preprocess + processors_.push_back(std::make_shared()); + + bool has_permute = false; + for (const auto& op : cfg["Preprocess"]) { + std::string op_name = op["type"].as(); + if (op_name == "NormalizeImage") { + auto mean = op["mean"].as>(); + auto std = op["std"].as>(); + bool is_scale = true; + if (op["is_scale"]) { + is_scale = op["is_scale"].as(); + } + std::string norm_type = "mean_std"; + if (op["norm_type"]) { + norm_type = op["norm_type"].as(); + } + if (norm_type != "mean_std") { + std::fill(mean.begin(), mean.end(), 0.0); + std::fill(std.begin(), std.end(), 1.0); + } + processors_.push_back(std::make_shared(mean, std, is_scale)); + } else if (op_name == "Resize") { + bool keep_ratio = op["keep_ratio"].as(); + auto target_size = op["target_size"].as>(); + int interp = op["interp"].as(); + FDASSERT(target_size.size() == 2, + "Require size of target_size be 2, but now it's %lu.", + target_size.size()); + if (!keep_ratio) { + int width = target_size[1]; + int height = target_size[0]; + processors_.push_back( + std::make_shared(width, height, -1.0, -1.0, interp, false)); + } else { + int min_target_size = std::min(target_size[0], target_size[1]); + int max_target_size = std::max(target_size[0], target_size[1]); + std::vector max_size; + if (max_target_size > 0) { + max_size.push_back(max_target_size); + max_size.push_back(max_target_size); + } + processors_.push_back(std::make_shared( + min_target_size, interp, true, max_size)); + } + } else if (op_name == "Permute") { + // Do nothing, do permute as the last operation + has_permute = true; + continue; + } else { + FDERROR << "Unexcepted preprocess operator: " << op_name << "." + << std::endl; + return false; + } + } + if (!disable_permute_) { + if (has_permute) { + // permute = cast + HWC2CHW + processors_.push_back(std::make_shared("float")); + processors_.push_back(std::make_shared()); + } + } + + // Fusion will improve performance + FuseTransforms(&processors_); + + input_k_data_ = cfg["k_data"].as>(); + input_ratio_data_ = cfg["ratio_data"].as>(); + return true; +} + +bool SmokePreprocessor::Apply(FDMatBatch* image_batch, + std::vector* outputs) { + if (image_batch->mats->empty()) { + FDERROR << "The size of input images should be greater than 0." + << std::endl; + return false; + } + if (!initialized_) { + FDERROR << "The preprocessor is not initialized." << std::endl; + return false; + } + // There are 3 outputs, image, k_data, ratio_data + outputs->resize(3); + int batch = static_cast(image_batch->mats->size()); + + // Allocate memory for k_data + (*outputs)[2].Resize({batch, 3, 3}, FDDataType::FP32); + + // Allocate memory for ratio_data + (*outputs)[0].Resize({batch, 2}, FDDataType::FP32); + + auto* k_data_ptr = reinterpret_cast((*outputs)[2].MutableData()); + + auto* ratio_data_ptr = reinterpret_cast((*outputs)[0].MutableData()); + + for (size_t i = 0; i < image_batch->mats->size(); ++i) { + FDMat* mat = &(image_batch->mats->at(i)); + for (size_t j = 0; j < processors_.size(); ++j) { + if (!(*(processors_[j].get()))(mat)) { + FDERROR << "Failed to processs image:" << i << " in " + << processors_[j]->Name() << "." << std::endl; + return false; + } + } + + memcpy(k_data_ptr + i * 9, input_k_data_.data(), 9 * sizeof(float)); + memcpy(ratio_data_ptr + i * 2, input_ratio_data_.data(), 2 * sizeof(float)); + } + + FDTensor* tensor = image_batch->Tensor(); + (*outputs)[1].SetExternalData(tensor->Shape(), tensor->Dtype(), + tensor->Data(), tensor->device, + tensor->device_id); + return true; +} + +} // namespace perception +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/perception/paddle3d/smoke/preprocessor.h b/fastdeploy/vision/perception/paddle3d/smoke/preprocessor.h new file mode 100755 index 0000000000..73c804914b --- /dev/null +++ b/fastdeploy/vision/perception/paddle3d/smoke/preprocessor.h @@ -0,0 +1,61 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/manager.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace perception { +/*! @brief Preprocessor object for Smoke serials model. + */ +class FASTDEPLOY_DECL SmokePreprocessor : public ProcessorManager { + public: + SmokePreprocessor() = default; + /** \brief Create a preprocessor instance for Smoke model + * + * \param[in] config_file Path of configuration file for deployment, e.g smoke/infer_cfg.yml + */ + explicit SmokePreprocessor(const std::string& config_file); + + /** \brief Process the input image and prepare input tensors for runtime + * + * \param[in] images The input image data list, all the elements are returned by cv::imread() + * \param[in] outputs The output tensors which will feed in runtime + * \param[in] ims_info The shape info list, record input_shape and output_shape + * \return true if the preprocess successed, otherwise false + */ + bool Apply(FDMatBatch* image_batch, std::vector* outputs); + + protected: + bool BuildPreprocessPipelineFromConfig(); + std::vector> processors_; + + bool disable_permute_ = false; + + bool initialized_ = false; + + std::string config_file_; + + std::vector input_k_data_; + + std::vector input_ratio_data_; +}; + +} // namespace perception +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/perception/paddle3d/smoke/smoke.cc b/fastdeploy/vision/perception/paddle3d/smoke/smoke.cc new file mode 100755 index 0000000000..90b98c720d --- /dev/null +++ b/fastdeploy/vision/perception/paddle3d/smoke/smoke.cc @@ -0,0 +1,82 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/perception/paddle3d/smoke/smoke.h" + +namespace fastdeploy { +namespace vision { +namespace perception { + +Smoke::Smoke(const std::string& model_file, const std::string& params_file, + const std::string& config_file, const RuntimeOption& custom_option, + const ModelFormat& model_format) + : preprocessor_(config_file) { + valid_cpu_backends = {Backend::PDINFER}; + valid_gpu_backends = {Backend::PDINFER}; + + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + initialized = Initialize(); +} + +bool Smoke::Initialize() { + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + return true; +} + +bool Smoke::Predict(const cv::Mat& im, PerceptionResult* result) { + std::vector results; + if (!BatchPredict({im}, &results)) { + return false; + } + if (results.size()) { + *result = std::move(results[0]); + } + return true; +} + +bool Smoke::BatchPredict(const std::vector& images, + std::vector* results) { + std::vector fd_images = WrapMat(images); + + if (!preprocessor_.Run(&fd_images, &reused_input_tensors_)) { + FDERROR << "Failed to preprocess the input image." << std::endl; + return false; + } + + reused_input_tensors_[0].name = InputInfoOfRuntime(0).name; + reused_input_tensors_[1].name = InputInfoOfRuntime(1).name; + reused_input_tensors_[2].name = InputInfoOfRuntime(2).name; + + if (!Infer(reused_input_tensors_, &reused_output_tensors_)) { + FDERROR << "Failed to inference by runtime." << std::endl; + return false; + } + + if (!postprocessor_.Run(reused_output_tensors_, results)) { + FDERROR << "Failed to postprocess the inference results by runtime." + << std::endl; + return false; + } + return true; +} + +} // namespace perception +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/perception/paddle3d/smoke/smoke.h b/fastdeploy/vision/perception/paddle3d/smoke/smoke.h new file mode 100755 index 0000000000..ebc88bbf20 --- /dev/null +++ b/fastdeploy/vision/perception/paddle3d/smoke/smoke.h @@ -0,0 +1,78 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. //NOLINT +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/perception/paddle3d/smoke/preprocessor.h" +#include "fastdeploy/vision/perception/paddle3d/smoke/postprocessor.h" + +namespace fastdeploy { +namespace vision { +namespace perception { +/*! @brief smoke model object used when to load a smoke model exported by smoke. + */ +class FASTDEPLOY_DECL Smoke : public FastDeployModel { + public: + /** \brief Set path of model file and the configuration of runtime. + * + * \param[in] model_file Path of model file, e.g smoke/model.pdiparams + * \param[in] params_file Path of parameter file, e.g smoke/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in "valid_cpu_backends" + * \param[in] model_format Model format of the loaded model, default is Paddle format + */ + Smoke(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + + std::string ModelName() const { return "Paddle3D/smoke"; } + + /** \brief Predict the perception result for an input image + * + * \param[in] img The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] result The output perception result will be writen to this structure + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(const cv::Mat& img, PerceptionResult* result); + + /** \brief Predict the perception results for a batch of input images + * + * \param[in] imgs, The input image list, each element comes from cv::imread() + * \param[in] results The output perception result list + * \return true if the prediction successed, otherwise false + */ + virtual bool BatchPredict(const std::vector& imgs, + std::vector* results); + + /// Get preprocessor reference of Smoke + virtual SmokePreprocessor& GetPreprocessor() { + return preprocessor_; + } + + /// Get postprocessor reference of Smoke + virtual SmokePostprocessor& GetPostprocessor() { + return postprocessor_; + } + + protected: + bool Initialize(); + SmokePreprocessor preprocessor_; + SmokePostprocessor postprocessor_; + bool initialized_ = false; +}; + +} // namespace perception +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/perception/paddle3d/smoke/smoke_pybind.cc b/fastdeploy/vision/perception/paddle3d/smoke/smoke_pybind.cc new file mode 100644 index 0000000000..03c6499d95 --- /dev/null +++ b/fastdeploy/vision/perception/paddle3d/smoke/smoke_pybind.cc @@ -0,0 +1,92 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindSmoke(pybind11::module& m) { + pybind11::class_(m, "SmokePreprocessor") + .def(pybind11::init()) + .def("run", [](vision::perception::SmokePreprocessor& self, + std::vector& im_list) { + std::vector images; + for (size_t i = 0; i < im_list.size(); ++i) { + images.push_back(vision::WrapMat(PyArrayToCvMat(im_list[i]))); + } + std::vector outputs; + if (!self.Run(&images, &outputs)) { + throw std::runtime_error( + "Failed to preprocess the input data in SmokePreprocessor."); + } + for (size_t i = 0; i < outputs.size(); ++i) { + outputs[i].StopSharing(); + } + return outputs; + }); + + pybind11::class_(m, + "SmokePostprocessor") + .def(pybind11::init<>()) + .def("run", + [](vision::perception::SmokePostprocessor& self, + std::vector& inputs) { + std::vector results; + if (!self.Run(inputs, &results)) { + throw std::runtime_error( + "Failed to postprocess the runtime result in " + "SmokePostprocessor."); + } + return results; + }) + .def("run", [](vision::perception::SmokePostprocessor& self, + std::vector& input_array) { + std::vector results; + std::vector inputs; + PyArrayToTensorList(input_array, &inputs, /*share_buffer=*/true); + if (!self.Run(inputs, &results)) { + throw std::runtime_error( + "Failed to postprocess the runtime result in " + "SmokePostprocessor."); + } + return results; + }); + + pybind11::class_(m, "Smoke") + .def(pybind11::init()) + .def("predict", + [](vision::perception::Smoke& self, pybind11::array& data) { + auto mat = PyArrayToCvMat(data); + vision::PerceptionResult res; + self.Predict(mat, &res); + return res; + }) + .def("batch_predict", + [](vision::perception::Smoke& self, + std::vector& data) { + std::vector images; + for (size_t i = 0; i < data.size(); ++i) { + images.push_back(PyArrayToCvMat(data[i])); + } + std::vector results; + self.BatchPredict(images, &results); + return results; + }) + .def_property_readonly("preprocessor", + &vision::perception::Smoke::GetPreprocessor) + .def_property_readonly("postprocessor", + &vision::perception::Smoke::GetPostprocessor); +} +} // namespace fastdeploy diff --git a/fastdeploy/vision/perception/perception_pybind.cc b/fastdeploy/vision/perception/perception_pybind.cc new file mode 100755 index 0000000000..6d433c91e4 --- /dev/null +++ b/fastdeploy/vision/perception/perception_pybind.cc @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { + +void BindSmoke(pybind11::module& m); + +void BindPerception(pybind11::module& m) { + auto perception_module = + m.def_submodule("perception", "3D object perception models."); + BindSmoke(perception_module); +} +} // namespace fastdeploy diff --git a/fastdeploy/vision/vision_pybind.cc b/fastdeploy/vision/vision_pybind.cc index 172d9c0981..4bc04886f6 100755 --- a/fastdeploy/vision/vision_pybind.cc +++ b/fastdeploy/vision/vision_pybind.cc @@ -33,6 +33,7 @@ void BindHeadPose(pybind11::module& m); void BindSR(pybind11::module& m); void BindGeneration(pybind11::module& m); void BindVisualize(pybind11::module& m); +void BindPerception(pybind11::module& m); void BindVision(pybind11::module& m) { pybind11::class_(m, "Mask") @@ -108,6 +109,40 @@ void BindVision(pybind11::module& m) { .def("__repr__", &vision::DetectionResult::Str) .def("__str__", &vision::DetectionResult::Str); + pybind11::class_(m, "PerceptionResult") + .def(pybind11::init()) + .def_readwrite("scores", &vision::PerceptionResult::scores) + .def_readwrite("label_ids", &vision::PerceptionResult::label_ids) + .def_readwrite("boxes", &vision::PerceptionResult::boxes) + .def_readwrite("center", &vision::PerceptionResult::center) + .def_readwrite("observation_angle", + &vision::PerceptionResult::observation_angle) + .def_readwrite("yaw_angle", &vision::PerceptionResult::yaw_angle) + .def_readwrite("velocity", &vision::PerceptionResult::velocity) + .def(pybind11::pickle( + [](const vision::PerceptionResult& d) { + return pybind11::make_tuple(d.scores, d.label_ids, d.boxes, + d.center, d.observation_angle, + d.yaw_angle, d.velocity); + }, + [](pybind11::tuple t) { + if (t.size() != 7) + throw std::runtime_error( + "vision::PerceptionResult pickle with Invalid state!"); + + vision::PerceptionResult d; + d.scores = t[0].cast>(); + d.label_ids = t[1].cast>(); + d.boxes = t[2].cast>>(); + d.center = t[3].cast>>(); + d.observation_angle = t[4].cast>(); + d.yaw_angle = t[5].cast>(); + d.velocity = t[6].cast>>(); + return d; + })) + .def("__repr__", &vision::PerceptionResult::Str) + .def("__str__", &vision::PerceptionResult::Str); + pybind11::class_(m, "OCRResult") .def(pybind11::init()) .def_readwrite("boxes", &vision::OCRResult::boxes) @@ -224,5 +259,6 @@ void BindVision(pybind11::module& m) { BindSR(m); BindGeneration(m); BindVisualize(m); + BindPerception(m); } } // namespace fastdeploy diff --git a/fastdeploy/vision/visualize/perception.cc b/fastdeploy/vision/visualize/perception.cc new file mode 100755 index 0000000000..f388c54461 --- /dev/null +++ b/fastdeploy/vision/visualize/perception.cc @@ -0,0 +1,194 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "fastdeploy/vision/visualize/visualize.h" +#include "opencv2/calib3d/calib3d.hpp" +#include "opencv2/imgproc/imgproc.hpp" +#include "yaml-cpp/yaml.h" + +namespace fastdeploy { +namespace vision { + +using matrix = std::vector>; + +matrix Multiple(const matrix a, const matrix b) { + const int m = a.size(); // a rows + if (m == 0) { + matrix c; + return c; + } + if (a[0].size() != b.size()) { + FDERROR << "A[m,n] * B[p,q], n must equal to p." << std::endl; + matrix c; + return c; + } + const int n = a[0].size(); // a cols + const int p = b[0].size(); // b cols + matrix c(m, std::vector(p, 0)); + for (auto i = 0; i < m; i++) { + for (auto j = 0; j < p; j++) { + for (auto k = 0; k < n; k++) c[i][j] += a[i][k] * b[k][j]; + } + } + return c; +} + +cv::Mat VisPerception(const cv::Mat& im, const PerceptionResult& result, + const std::string& config_file, float score_threshold, + int line_size, float font_size) { + if (result.scores.empty()) { + return im; + } + YAML::Node cfg; + try { + cfg = YAML::LoadFile(config_file); + } catch (YAML::BadFile& e) { + FDERROR << "Failed to load yaml file " << config_file + << ", maybe you should check this file." << std::endl; + return im; + } + + std::vector target_size; + for (const auto& op : cfg["Preprocess"]) { + std::string op_name = op["type"].as(); + if (op_name == "Resize") { + target_size = op["target_size"].as>(); + } + } + + std::vector vec_k_data = cfg["k_data"].as>(); + if (vec_k_data.size() != 9) { + FDERROR + << "The K data load from the yaml file: " << config_file + << " is unexpected, the expected size is 9, but the loaded size is: " + << vec_k_data.size() << " ,maybe you should check this file." + << std::endl; + return im; + } + matrix k_data(3, std::vector()); + for (auto j = 0; j < 3; j++) { + k_data[j].insert(k_data[j].begin(), vec_k_data.begin() + j * 3, + vec_k_data.begin() + j * 3 + 3); + } + + std::vector rvec = {1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0}; + std::vector tvec = {0, 0, 0}; + + matrix connect_line_id = {{1, 0}, {2, 7}, {3, 6}, {4, 5}, {1, 2}, {2, 3}, + {3, 4}, {4, 1}, {0, 7}, {7, 6}, {6, 5}, {5, 0}}; + + int max_label_id = + *std::max_element(result.label_ids.begin(), result.label_ids.end()); + std::vector color_map = GenerateColorMap(max_label_id); + int h = im.rows; + int w = im.cols; + cv::Mat vis_im = im.clone(); + cv::resize(im, vis_im, cv::Size(target_size[1], target_size[0]), 0, 0, 0); + for (size_t i = 0; i < result.scores.size(); ++i) { + if (result.scores[i] < 0.5) { + continue; + } + float h = result.boxes[i][4]; + float w = result.boxes[i][5]; + float l = result.boxes[i][6]; + + float x = result.center[i][0]; + float y = result.center[i][1]; + float z = result.center[i][2]; + std::vector x_corners = {0, l, l, l, l, 0, 0, 0}; + std::vector y_corners = {0, 0, h, h, 0, 0, h, h}; + std::vector z_corners = {0, 0, 0, w, w, w, w, 0}; + + for (auto j = 0; j < x_corners.size(); j++) { + x_corners[j] = x_corners[j] - l / 2; + y_corners[j] = y_corners[j] - h; + z_corners[j] = z_corners[j] - w / 2; + } + + matrix corners_3d = {x_corners, y_corners, z_corners}; + + float ry = result.yaw_angle[i]; + matrix rot_mat = { + {cosf(ry), 0, sinf(ry)}, {0, 1, 0}, {sinf(ry), 0, cosf(ry)}}; + + matrix rot_corners_3d = Multiple(rot_mat, corners_3d); + + for (auto j = 0; j < rot_corners_3d[0].size(); j++) { + rot_corners_3d[0][j] += x; + rot_corners_3d[1][j] += y; + rot_corners_3d[2][j] += z; + } + + auto corners_2d = Multiple(k_data, rot_corners_3d); + + for (auto j = 0; j < corners_2d[0].size(); j++) { + corners_2d[0][j] /= corners_2d[2][j]; + corners_2d[1][j] /= corners_2d[2][j]; + } + + std::vector box2d = { + *std::min_element(corners_2d[0].begin(), corners_2d[0].end()), + *std::min_element(corners_2d[1].begin(), corners_2d[1].end()), + *std::max_element(corners_2d[0].begin(), corners_2d[0].end()), + *std::max_element(corners_2d[1].begin(), corners_2d[1].end())}; + + if (box2d[0] == 0 && box2d[1] == 0 && box2d[2] == 0 && box2d[3] == 0) { + continue; + } + + std::vector points3d; + for (auto j = 0; j < rot_corners_3d[0].size(); j++) { + points3d.push_back(cv::Point3f(rot_corners_3d[0][j], rot_corners_3d[1][j], + rot_corners_3d[2][j])); + } + cv::Mat rVec(3, 3, cv::DataType::type, rvec.data()); + cv::Mat tVec(3, 1, cv::DataType::type, tvec.data()); + std::vector vec_k; + for (auto&& v : k_data) { + vec_k.insert(vec_k.end(), v.begin(), v.end()); + } + cv::Mat intrinsicMat(3, 3, cv::DataType::type, vec_k.data()); + cv::Mat distCoeffs(5, 1, cv::DataType::type); + std::vector projectedPoints; + cv::projectPoints(points3d, rVec, tVec, intrinsicMat, distCoeffs, + projectedPoints); + + int c0 = color_map[3 * result.label_ids[i] + 0]; + int c1 = color_map[3 * result.label_ids[i] + 1]; + int c2 = color_map[3 * result.label_ids[i] + 2]; + cv::Scalar color = cv::Scalar(c0, c1, c2); + for (auto id = 0; id < connect_line_id.size(); id++) { + int p1 = connect_line_id[id][0]; + int p2 = connect_line_id[id][1]; + cv::line(vis_im, projectedPoints[p1], projectedPoints[p2], color, 1); + } + int font = cv::FONT_HERSHEY_SIMPLEX; + std::string score = std::to_string(result.scores[i]); + if (score.size() > 4) { + score = score.substr(0, 4); + } + std::string text = std::to_string(result.label_ids[i]) + ", " + score; + cv::Point2f original; + original.x = box2d[0]; + original.y = box2d[1]; + cv::putText(vis_im, text, original, font, font_size, + cv::Scalar(255, 255, 255), 1); + } + return vis_im; +} + +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/visualize/visualize.h b/fastdeploy/vision/visualize/visualize.h index f382818c6a..c2d168b6ed 100755 --- a/fastdeploy/vision/visualize/visualize.h +++ b/fastdeploy/vision/visualize/visualize.h @@ -32,6 +32,12 @@ class FASTDEPLOY_DECL Visualize { static cv::Mat VisDetection(const cv::Mat& im, const DetectionResult& result, float score_threshold = 0.0, int line_size = 1, float font_size = 0.5f); + static cv::Mat VisPerception(const cv::Mat& im, + const PerceptionResult& result, + const std::string & config_file, + float score_threshold = 0.0, + int line_size = 1, + float font_size = 0.5f); static cv::Mat VisFaceDetection(const cv::Mat& im, const FaceDetectionResult& result, int line_size = 1, float font_size = 0.5f); @@ -82,6 +88,22 @@ FASTDEPLOY_DECL cv::Mat VisDetection(const cv::Mat& im, const std::vector& labels, float score_threshold = 0.0, int line_size = 1, float font_size = 0.5f); +/** \brief Show the visualized results with custom labels for detection models + * + * \param[in] im the input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] result the result produced by model + * \param[in] labels the visualized result will show the bounding box contain class label + * \param[in] score_threshold threshold for result scores, the bounding box will not be shown if the score is less than score_threshold + * \param[in] line_size line size for bounding boxes + * \param[in] font_size font size for text + * \return cv::Mat type stores the visualized results + */ +FASTDEPLOY_DECL cv::Mat VisPerception(const cv::Mat& im, + const PerceptionResult& result, + const std::string & config_file, + float score_threshold = 0.0, + int line_size = 1 , + float font_size = 0.5f); /** \brief Show the visualized results for classification models * * \param[in] im the input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format diff --git a/fastdeploy/vision/visualize/visualize_pybind.cc b/fastdeploy/vision/visualize/visualize_pybind.cc index ca13f4ffaf..f5d9799d01 100644 --- a/fastdeploy/vision/visualize/visualize_pybind.cc +++ b/fastdeploy/vision/visualize/visualize_pybind.cc @@ -33,6 +33,18 @@ void BindVisualize(pybind11::module& m) { vision::Mat(vis_im).ShareWithTensor(&out); return TensorToPyArray(out); }) + .def("vis_perception", + [](pybind11::array& im_data, vision::PerceptionResult& result, + const std::string& config_file, float score_threshold, + int line_size, float font_size) { + auto im = PyArrayToCvMat(im_data); + auto vis_im = + vision::VisPerception(im, result, config_file, score_threshold, + line_size, font_size); + FDTensor out; + vision::Mat(vis_im).ShareWithTensor(&out); + return TensorToPyArray(out); + }) .def("vis_face_detection", [](pybind11::array& im_data, vision::FaceDetectionResult& result, int line_size, float font_size) { diff --git a/python/fastdeploy/runtime.py b/python/fastdeploy/runtime.py index f7480de4b3..78d3dfe3d8 100755 --- a/python/fastdeploy/runtime.py +++ b/python/fastdeploy/runtime.py @@ -484,6 +484,29 @@ def set_trt_input_shape(self, return self._option.trt_option.set_shape(tensor_name, min_shape, opt_shape, max_shape) + def set_trt_input_data(self, + tensor_name, + min_input_data, + opt_input_data=None, + max_input_data=None): + """Set input data while using TensorRT backend with loadding a model contains dynamic input shape. + + :param tensor_name: (str)Name of input which has dynamic shape + :param min_input_data: (list of int)Input data for Minimum shape of the input. + :param opt_input_data: (list of int)Input data for Optimize shape of the input, if set to None, it will keep same with min_input_data + :param max_input_data: (list of int)Input data for Maximum shape of the input, if set to None, it will keep same with the min_input_data + """ + logging.warning( + "`RuntimeOption.set_trt_input_data` will be deprecated in v1.2.0, please use `RuntimeOption.trt_option.set_input_data()` instead." + ) + if opt_input_data is None and max_input_data is None: + opt_input_data = min_input_data + opt_input_data = min_input_data + else: + assert opt_input_data is not None and max_input_data is not None, "Set min_input_data only, or set min_input_data, opt_input_data, max_input_data both." + return self._option.trt_option.set_input_data( + tensor_name, min_input_data, opt_input_data, max_input_data) + def set_trt_cache_file(self, cache_file_path): """Set a cache file path while using TensorRT backend. While loading a Paddle/ONNX model with set_trt_cache_file("./tensorrt_cache/model.trt"), if file `./tensorrt_cache/model.trt` exists, it will skip building tensorrt engine and load the cache file directly; if file `./tensorrt_cache/model.trt` doesn't exist, it will building tensorrt engine and save the engine as binary string to the cache file. diff --git a/python/fastdeploy/vision/__init__.py b/python/fastdeploy/vision/__init__.py index ba9a2d0ca9..01cce8ef25 100755 --- a/python/fastdeploy/vision/__init__.py +++ b/python/fastdeploy/vision/__init__.py @@ -27,6 +27,7 @@ from . import sr from . import evaluation from . import generation +from . import perception from .utils import fd_result_to_json from .visualize import * from .. import C diff --git a/python/fastdeploy/vision/detection/ppdet/__init__.py b/python/fastdeploy/vision/detection/ppdet/__init__.py old mode 100644 new mode 100755 diff --git a/python/fastdeploy/vision/perception/__init__.py b/python/fastdeploy/vision/perception/__init__.py new file mode 100755 index 0000000000..d8397af069 --- /dev/null +++ b/python/fastdeploy/vision/perception/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from .paddle3d.smoke import * diff --git a/python/fastdeploy/vision/perception/paddle3d/__init__.py b/python/fastdeploy/vision/perception/paddle3d/__init__.py new file mode 100644 index 0000000000..8034e10bfc --- /dev/null +++ b/python/fastdeploy/vision/perception/paddle3d/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import diff --git a/python/fastdeploy/vision/perception/paddle3d/smoke.py b/python/fastdeploy/vision/perception/paddle3d/smoke.py new file mode 100755 index 0000000000..030068d322 --- /dev/null +++ b/python/fastdeploy/vision/perception/paddle3d/smoke.py @@ -0,0 +1,106 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +import logging +from .... import FastDeployModel, ModelFormat +from .... import c_lib_wrap as C + + +class SmokePreprocessor: + def __init__(self, config_file): + """Create a preprocessor for Smoke + """ + self._preprocessor = C.vision.perception.SmokePreprocessor(config_file) + + def run(self, input_ims): + """Preprocess input images for Smoke + + :param: input_ims: (list of numpy.ndarray)The input image + :return: list of FDTensor + """ + return self._preprocessor.run(input_ims) + + +class SmokePostprocessor: + def __init__(self): + """Create a postprocessor for Smoke + """ + self._postprocessor = C.vision.perception.SmokePostprocessor() + + def run(self, runtime_results): + """Postprocess the runtime results for Smoke + + :param: runtime_results: (list of FDTensor)The output FDTensor results from runtime + :return: list of PerceptionResult(If the runtime_results is predict by batched samples, the length of this list equals to the batch size) + """ + return self._postprocessor.run(runtime_results) + + +class Smoke(FastDeployModel): + def __init__(self, + model_file, + params_file, + config_file, + runtime_option=None, + model_format=ModelFormat.PADDLE): + """Load a SMoke model exported by Smoke. + + :param model_file: (str)Path of model file, e.g ./smoke.pdmodel + :param params_file: (str)Path of parameters file, e.g ./smoke.pdiparams + :param config_file: (str)Path of config file, e.g ./infer_cfg.yaml + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + super(Smoke, self).__init__(runtime_option) + + self._model = C.vision.perception.Smoke( + model_file, params_file, config_file, self._runtime_option, + model_format) + assert self.initialized, "Smoke initialize failed." + + def predict(self, input_image): + """Detect an input image + + :param input_image: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :param conf_threshold: confidence threshold for postprocessing, default is 0.25 + :param nms_iou_threshold: iou threshold for NMS, default is 0.5 + :return: PerceptionResult + """ + return self._model.predict(input_image) + + def batch_predict(self, images): + """Classify a batch of input image + + :param im: (list of numpy.ndarray) The input image list, each element is a 3-D array with layout HWC, BGR format + :return list of PerceptionResult + """ + + return self._model.batch_predict(images) + + @property + def preprocessor(self): + """Get SmokePreprocessor object of the loaded model + + :return SmokePreprocessor + """ + return self._model.preprocessor + + @property + def postprocessor(self): + """Get SmokePostprocessor object of the loaded model + + :return SmokePostprocessor + """ + return self._model.postprocessor diff --git a/python/fastdeploy/vision/utils.py b/python/fastdeploy/vision/utils.py old mode 100644 new mode 100755 index f4e06cf274..bb40890cef --- a/python/fastdeploy/vision/utils.py +++ b/python/fastdeploy/vision/utils.py @@ -38,6 +38,19 @@ def detection_to_json(result): return json.dumps(r_json) +def perception_to_json(result): + r_json = { + "scores": result.scores, + "label_ids": result.label_ids, + "boxes": result.boxes, + "center": result.center, + "observation_angle": result.observation_angle, + "yaw_angle": result.yaw_angle, + "velocity": result.velocity + } + return json.dumps(r_json) + + def classify_to_json(result): r_json = { "label_ids": result.label_ids, @@ -151,6 +164,8 @@ def fd_result_to_json(result): return matting_to_json(result) elif isinstance(result, C.vision.HeadPoseResult): return head_pose_to_json(result) + elif isinstance(result, C.vision.PerceptionResult): + return perception_to_json(result) else: assert False, "{} Conversion to JSON format is not supported".format( type(result)) @@ -177,6 +192,18 @@ def json_to_detection(result): return det_result +def json_to_perception(result): + perception_result = C.vision.PerceptionResult() + perception_result.scores = result['scores'] + perception_result.label_ids = result['label_ids'] + perception_result.boxes = result['boxes'] + perception_result.center = result['center'] + perception_result.observation_angle = result['observation_angle'] + perception_result.yaw_angle = result['yaw_angle'] + perception_result.velocity = result['velocity'] + return perception_result + + def json_to_classify(result): cls_result = C.vision.ClassifyResult() cls_result.label_ids = result['label_ids'] diff --git a/python/fastdeploy/vision/visualize/__init__.py b/python/fastdeploy/vision/visualize/__init__.py index b87a2dfda0..930f49376c 100755 --- a/python/fastdeploy/vision/visualize/__init__.py +++ b/python/fastdeploy/vision/visualize/__init__.py @@ -38,6 +38,26 @@ def vis_detection(im_data, line_size, font_size) +def vis_perception(im_data, + det_result, + config_file, + score_threshold=0.0, + line_size=1, + font_size=0.5): + """Show the visualized results for 3d detection models + + :param im_data: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :param det_result: the result produced by model + :param config_file: the config file for detection and visualization + :param score_threshold: (float) score_threshold threshold for result scores, the bounding box will not be shown if the score is less than score_threshold + :param line_size: (float) line_size line size for bounding boxes + :param font_size: (float) font_size font size for text + :return: (numpy.ndarray) image with visualized results + """ + return C.vision.vis_perception(im_data, det_result, config_file, + score_threshold, line_size, font_size) + + def vis_keypoint_detection(im_data, keypoint_det_result, conf_threshold=0.5): """Show the visualized results for keypoint detection models diff --git a/python/scripts/process_libraries.py.in b/python/scripts/process_libraries.py.in old mode 100644 new mode 100755 index e30e7a92df..ffe5cac9e6 --- a/python/scripts/process_libraries.py.in +++ b/python/scripts/process_libraries.py.in @@ -46,7 +46,7 @@ def process_on_linux(current_dir): if len(items) != 4: os.remove(os.path.join(root, f)) continue - if items[0].strip() not in ["libopencv_highgui", "libopencv_video", "libopencv_videoio", "libopencv_imgcodecs", "libopencv_imgproc", "libopencv_core"]: + if items[0].strip() not in ["libopencv_highgui", "libopencv_video", "libopencv_videoio", "libopencv_imgcodecs", "libopencv_imgproc", "libopencv_core", "libopencv_calib3d", "libopencv_features2d", "libopencv_flann"]: os.remove(os.path.join(root, f)) all_libs_paths = [third_libs_path] + user_specified_dirs