diff --git a/examples/vision/detection/paddledetection/cpp/infer_faster_rcnn.cc b/examples/vision/detection/paddledetection/cpp/infer_faster_rcnn.cc index 6471b457e3..52d49d21e1 100644 --- a/examples/vision/detection/paddledetection/cpp/infer_faster_rcnn.cc +++ b/examples/vision/detection/paddledetection/cpp/infer_faster_rcnn.cc @@ -24,8 +24,10 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "infer_cfg.yml"; + auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto model = fastdeploy::vision::detection::FasterRCNN( - model_file, params_file, config_file); + model_file, params_file, config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; diff --git a/examples/vision/detection/paddledetection/cpp/infer_mask_rcnn.cc b/examples/vision/detection/paddledetection/cpp/infer_mask_rcnn.cc index e8d1dac87b..39e3eb81f2 100644 --- a/examples/vision/detection/paddledetection/cpp/infer_mask_rcnn.cc +++ b/examples/vision/detection/paddledetection/cpp/infer_mask_rcnn.cc @@ -24,8 +24,10 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "infer_cfg.yml"; + auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto model = fastdeploy::vision::detection::MaskRCNN(model_file, params_file, - config_file); + config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; diff --git a/examples/vision/detection/paddledetection/cpp/infer_picodet.cc b/examples/vision/detection/paddledetection/cpp/infer_picodet.cc index 9c167c85dc..9ecd49e023 100644 --- a/examples/vision/detection/paddledetection/cpp/infer_picodet.cc +++ b/examples/vision/detection/paddledetection/cpp/infer_picodet.cc @@ -24,8 +24,10 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "infer_cfg.yml"; + auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto model = fastdeploy::vision::detection::PicoDet(model_file, params_file, - config_file); + config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; diff --git a/examples/vision/detection/paddledetection/cpp/infer_ppyolo.cc b/examples/vision/detection/paddledetection/cpp/infer_ppyolo.cc index 47946c8fd6..ab2a291d01 100644 --- a/examples/vision/detection/paddledetection/cpp/infer_ppyolo.cc +++ b/examples/vision/detection/paddledetection/cpp/infer_ppyolo.cc @@ -24,8 +24,10 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "infer_cfg.yml"; + auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto model = fastdeploy::vision::detection::PPYOLO(model_file, params_file, - config_file); + config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; diff --git a/examples/vision/detection/paddledetection/cpp/infer_ppyoloe.cc b/examples/vision/detection/paddledetection/cpp/infer_ppyoloe.cc index ed2dfd2bd1..2e0f8dc772 100644 --- a/examples/vision/detection/paddledetection/cpp/infer_ppyoloe.cc +++ b/examples/vision/detection/paddledetection/cpp/infer_ppyoloe.cc @@ -24,8 +24,10 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "infer_cfg.yml"; + auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto model = fastdeploy::vision::detection::PPYOLOE(model_file, params_file, - config_file); + config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; diff --git a/examples/vision/detection/paddledetection/cpp/infer_yolov3.cc b/examples/vision/detection/paddledetection/cpp/infer_yolov3.cc index 5945e86f29..e58c71e852 100644 --- a/examples/vision/detection/paddledetection/cpp/infer_yolov3.cc +++ b/examples/vision/detection/paddledetection/cpp/infer_yolov3.cc @@ -24,8 +24,10 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "infer_cfg.yml"; + auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto model = fastdeploy::vision::detection::YOLOv3(model_file, params_file, - config_file); + config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; diff --git a/examples/vision/detection/paddledetection/cpp/infer_yolox.cc b/examples/vision/detection/paddledetection/cpp/infer_yolox.cc index 1c7f677421..2dd4cefed4 100644 --- a/examples/vision/detection/paddledetection/cpp/infer_yolox.cc +++ b/examples/vision/detection/paddledetection/cpp/infer_yolox.cc @@ -24,8 +24,10 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "infer_cfg.yml"; + auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto model = fastdeploy::vision::detection::PaddleYOLOX( - model_file, params_file, config_file); + model_file, params_file, config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; diff --git a/examples/vision/keypointdetection/tiny_pose/cpp/pptinypose_infer.cc b/examples/vision/keypointdetection/tiny_pose/cpp/pptinypose_infer.cc index 30587dbfb4..87792e6f5a 100644 --- a/examples/vision/keypointdetection/tiny_pose/cpp/pptinypose_infer.cc +++ b/examples/vision/keypointdetection/tiny_pose/cpp/pptinypose_infer.cc @@ -25,8 +25,10 @@ void CpuInfer(const std::string& tinypose_model_dir, auto tinypose_model_file = tinypose_model_dir + sep + "model.pdmodel"; auto tinypose_params_file = tinypose_model_dir + sep + "model.pdiparams"; auto tinypose_config_file = tinypose_model_dir + sep + "infer_cfg.yml"; + auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto tinypose_model = fastdeploy::vision::keypointdetection::PPTinyPose( - tinypose_model_file, tinypose_params_file, tinypose_config_file); + tinypose_model_file, tinypose_params_file, tinypose_config_file, option); if (!tinypose_model.Initialized()) { std::cerr << "TinyPose Model Failed to initialize." << std::endl; return; diff --git a/examples/vision/matting/ppmatting/cpp/infer.cc b/examples/vision/matting/ppmatting/cpp/infer.cc index 4dd9eb1845..2c89c89e0e 100644 --- a/examples/vision/matting/ppmatting/cpp/infer.cc +++ b/examples/vision/matting/ppmatting/cpp/infer.cc @@ -26,6 +26,7 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file, auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "deploy.yaml"; auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto model = fastdeploy::vision::matting::PPMatting(model_file, params_file, config_file, option); if (!model.Initialized()) { diff --git a/examples/vision/segmentation/paddleseg/cpp/infer.cc b/examples/vision/segmentation/paddleseg/cpp/infer.cc index b953db63ef..37f16f04eb 100644 --- a/examples/vision/segmentation/paddleseg/cpp/infer.cc +++ b/examples/vision/segmentation/paddleseg/cpp/infer.cc @@ -24,8 +24,10 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "deploy.yaml"; + auto option = fastdeploy::RuntimeOption(); + option.UseCpu(); auto model = fastdeploy::vision::segmentation::PaddleSegModel( - model_file, params_file, config_file); + model_file, params_file, config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; diff --git a/fastdeploy/backends/lite/lite_backend.cc b/fastdeploy/backends/lite/lite_backend.cc index 6895958653..be3ec17a7f 100644 --- a/fastdeploy/backends/lite/lite_backend.cc +++ b/fastdeploy/backends/lite/lite_backend.cc @@ -42,7 +42,11 @@ FDDataType LiteDataTypeToFD(const paddle::lite_api::PrecisionType& dtype) { void LiteBackend::BuildOption(const LiteBackendOption& option) { option_ = option; std::vector valid_places; - if (option.enable_fp16) { + if (option_.enable_int8) { + valid_places.push_back( + paddle::lite_api::Place{TARGET(kARM), PRECISION(kInt8)}); + } + if (option_.enable_fp16) { paddle::lite_api::MobileConfig check_fp16_config; // Determine whether the device supports the FP16 // instruction set (or whether it is an arm device @@ -58,12 +62,12 @@ void LiteBackend::BuildOption(const LiteBackendOption& option) { valid_places.push_back( paddle::lite_api::Place{TARGET(kARM), PRECISION(kFloat)}); config_.set_valid_places(valid_places); - if (option.threads > 0) { - config_.set_threads(option.threads); + if (option_.threads > 0) { + config_.set_threads(option_.threads); } - if (option.power_mode > 0) { + if (option_.power_mode > 0) { config_.set_power_mode( - static_cast(option.power_mode)); + static_cast(option_.power_mode)); } } @@ -136,14 +140,13 @@ TensorInfo LiteBackend::GetOutputInfo(int index) { std::vector LiteBackend::GetOutputInfos() { return outputs_desc_; } bool LiteBackend::Infer(std::vector& inputs, - std::vector* outputs) { + std::vector* outputs) { if (inputs.size() != inputs_desc_.size()) { FDERROR << "[LiteBackend] Size of inputs(" << inputs.size() << ") should keep same with the inputs of this model(" << inputs_desc_.size() << ")." << std::endl; return false; } - for (size_t i = 0; i < inputs.size(); ++i) { auto iter = inputs_order_.find(inputs[i].name); if (iter == inputs_order_.end()) { @@ -152,12 +155,29 @@ bool LiteBackend::Infer(std::vector& inputs, return false; } auto tensor = predictor_->GetInput(iter->second); - tensor->Resize(inputs[i].shape); - tensor->ShareExternalMemory(const_cast(inputs[i].CpuData()), - inputs[i].Nbytes(), - paddle::lite_api::TargetType::kARM); + // Adjust dims only, allocate lazy. + tensor->Resize(inputs[i].shape); + if (inputs[i].dtype == FDDataType::FP32) { + tensor->CopyFromCpu( + reinterpret_cast(const_cast( + inputs[i].CpuData()))); + } else if (inputs[i].dtype == FDDataType::INT32) { + tensor->CopyFromCpu( + reinterpret_cast(const_cast( + inputs[i].CpuData()))); + } else if (inputs[i].dtype == FDDataType::INT8) { + tensor->CopyFromCpu( + reinterpret_cast(const_cast( + inputs[i].CpuData()))); + } else if (inputs[i].dtype == FDDataType::UINT8) { + tensor->CopyFromCpu( + reinterpret_cast(const_cast( + inputs[i].CpuData()))); + } else { + FDASSERT(false, "Unexpected data type of %d.", inputs[i].dtype); + } } - + predictor_->Run(); outputs->resize(outputs_desc_.size()); diff --git a/fastdeploy/backends/lite/lite_backend.h b/fastdeploy/backends/lite/lite_backend.h index cf88390a43..fa18cfe47e 100644 --- a/fastdeploy/backends/lite/lite_backend.h +++ b/fastdeploy/backends/lite/lite_backend.h @@ -37,6 +37,8 @@ struct LiteBackendOption { int power_mode = 3; // enable fp16 bool enable_fp16 = false; + // enable int8 + bool enable_int8 = false; // optimized model dir for CxxConfig std::string optimized_model_dir = ""; // TODO(qiuyanjun): support more options for lite backend. diff --git a/fastdeploy/runtime.cc b/fastdeploy/runtime.cc index 087b57755d..e0b3e59d95 100755 --- a/fastdeploy/runtime.cc +++ b/fastdeploy/runtime.cc @@ -321,7 +321,17 @@ void RuntimeOption::EnableLiteFP16() { lite_enable_fp16 = true; } -void RuntimeOption::DisableLiteFP16() { lite_enable_fp16 = false; } +void RuntimeOption::DisableLiteFP16() { + lite_enable_fp16 = false; +} + +void RuntimeOption::EnableLiteInt8() { + lite_enable_int8 = true; +} + +void RuntimeOption::DisableLiteInt8() { + lite_enable_int8 = false; +} void RuntimeOption::SetLitePowerMode(LitePowerMode mode) { lite_power_mode = mode; @@ -650,6 +660,7 @@ void Runtime::CreateLiteBackend() { #ifdef ENABLE_LITE_BACKEND auto lite_option = LiteBackendOption(); lite_option.threads = option.cpu_thread_num; + lite_option.enable_int8 = option.lite_enable_int8; lite_option.enable_fp16 = option.lite_enable_fp16; lite_option.power_mode = static_cast(option.lite_power_mode); lite_option.optimized_model_dir = option.lite_optimized_model_dir; diff --git a/fastdeploy/runtime.h b/fastdeploy/runtime.h index 26628217bb..7cb4f0d4ef 100755 --- a/fastdeploy/runtime.h +++ b/fastdeploy/runtime.h @@ -173,6 +173,16 @@ struct FASTDEPLOY_DECL RuntimeOption { */ void DisableLiteFP16(); + /** + * @brief enable int8 precision while use paddle lite backend + */ + void EnableLiteInt8(); + + /** + * @brief disable int8 precision, change to full precision(float32) + */ + void DisableLiteInt8(); + /** * @brief Set power mode while using Paddle Lite as inference backend, mode(0: LITE_POWER_HIGH; 1: LITE_POWER_LOW; 2: LITE_POWER_FULL; 3: LITE_POWER_NO_BIND, 4: LITE_POWER_RAND_HIGH; 5: LITE_POWER_RAND_LOW, refer [paddle lite](https://paddle-lite.readthedocs.io/zh/latest/api_reference/cxx_api_doc.html#set-power-mode) for more details) */ @@ -260,6 +270,8 @@ struct FASTDEPLOY_DECL RuntimeOption { // 3: LITE_POWER_NO_BIND 4: LITE_POWER_RAND_HIGH // 5: LITE_POWER_RAND_LOW LitePowerMode lite_power_mode = LitePowerMode::LITE_POWER_NO_BIND; + // enable int8 or not + bool lite_enable_int8 = false; // enable fp16 or not bool lite_enable_fp16 = false; // optimized model dir for CxxConfig diff --git a/fastdeploy/vision/detection/ppdet/mask_rcnn.cc b/fastdeploy/vision/detection/ppdet/mask_rcnn.cc index 60b92d6888..7c656c6695 100644 --- a/fastdeploy/vision/detection/ppdet/mask_rcnn.cc +++ b/fastdeploy/vision/detection/ppdet/mask_rcnn.cc @@ -24,7 +24,7 @@ MaskRCNN::MaskRCNN(const std::string& model_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::PDINFER}; + valid_cpu_backends = {Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/detection/ppdet/ppyolo.cc b/fastdeploy/vision/detection/ppdet/ppyolo.cc index befa16e18a..f0965e5f48 100644 --- a/fastdeploy/vision/detection/ppdet/ppyolo.cc +++ b/fastdeploy/vision/detection/ppdet/ppyolo.cc @@ -23,7 +23,7 @@ PPYOLO::PPYOLO(const std::string& model_file, const std::string& params_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER}; + valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER}; has_nms_ = true; runtime_option = custom_option; diff --git a/fastdeploy/vision/detection/ppdet/ppyoloe.cc b/fastdeploy/vision/detection/ppdet/ppyoloe.cc index 299b1b6651..99f1fc3928 100644 --- a/fastdeploy/vision/detection/ppdet/ppyoloe.cc +++ b/fastdeploy/vision/detection/ppdet/ppyoloe.cc @@ -14,7 +14,7 @@ PPYOLOE::PPYOLOE(const std::string& model_file, const std::string& params_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, Backend::PDINFER}; + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/detection/ppdet/rcnn.cc b/fastdeploy/vision/detection/ppdet/rcnn.cc index 709c8b6432..53cbffa568 100644 --- a/fastdeploy/vision/detection/ppdet/rcnn.cc +++ b/fastdeploy/vision/detection/ppdet/rcnn.cc @@ -24,7 +24,7 @@ FasterRCNN::FasterRCNN(const std::string& model_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::PDINFER}; + valid_cpu_backends = {Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER}; has_nms_ = true; runtime_option = custom_option; diff --git a/fastdeploy/vision/detection/ppdet/yolov3.cc b/fastdeploy/vision/detection/ppdet/yolov3.cc index 3ebcfe49f8..bcfb3aef9c 100644 --- a/fastdeploy/vision/detection/ppdet/yolov3.cc +++ b/fastdeploy/vision/detection/ppdet/yolov3.cc @@ -23,7 +23,7 @@ YOLOv3::YOLOv3(const std::string& model_file, const std::string& params_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, Backend::PDINFER}; + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/detection/ppdet/yolox.cc b/fastdeploy/vision/detection/ppdet/yolox.cc index 9401a30d82..f7405d4de3 100644 --- a/fastdeploy/vision/detection/ppdet/yolox.cc +++ b/fastdeploy/vision/detection/ppdet/yolox.cc @@ -24,7 +24,7 @@ PaddleYOLOX::PaddleYOLOX(const std::string& model_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::ORT, Backend::PDINFER}; + valid_cpu_backends = {Backend::ORT, Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/faceid/contrib/insightface_rec.cc b/fastdeploy/vision/faceid/contrib/insightface_rec.cc index 063a084318..cc4f1fceaa 100644 --- a/fastdeploy/vision/faceid/contrib/insightface_rec.cc +++ b/fastdeploy/vision/faceid/contrib/insightface_rec.cc @@ -30,7 +30,7 @@ InsightFaceRecognitionModel::InsightFaceRecognitionModel( valid_cpu_backends = {Backend::ORT}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; } runtime_option = custom_option; diff --git a/fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc b/fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc index 9bca72532b..94b7708f38 100644 --- a/fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc +++ b/fastdeploy/vision/keypointdet/pptinypose/pptinypose.cc @@ -16,7 +16,7 @@ PPTinyPose::PPTinyPose(const std::string& model_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/matting/ppmatting/ppmatting.cc b/fastdeploy/vision/matting/ppmatting/ppmatting.cc index e760ab5230..5fa63a48dd 100755 --- a/fastdeploy/vision/matting/ppmatting/ppmatting.cc +++ b/fastdeploy/vision/matting/ppmatting/ppmatting.cc @@ -25,7 +25,7 @@ PPMatting::PPMatting(const std::string& model_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::ORT, Backend::PDINFER}; + valid_cpu_backends = {Backend::ORT, Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/ocr/ppocr/classifier.cc b/fastdeploy/vision/ocr/ppocr/classifier.cc index 13fd69a07e..1fbd4cc364 100644 --- a/fastdeploy/vision/ocr/ppocr/classifier.cc +++ b/fastdeploy/vision/ocr/ppocr/classifier.cc @@ -30,7 +30,7 @@ Classifier::Classifier(const std::string& model_file, Backend::OPENVINO}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; } runtime_option = custom_option; diff --git a/fastdeploy/vision/ocr/ppocr/dbdetector.cc b/fastdeploy/vision/ocr/ppocr/dbdetector.cc index 09309aec10..8ee44fddfa 100644 --- a/fastdeploy/vision/ocr/ppocr/dbdetector.cc +++ b/fastdeploy/vision/ocr/ppocr/dbdetector.cc @@ -30,7 +30,7 @@ DBDetector::DBDetector(const std::string& model_file, Backend::OPENVINO}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; } diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v2.cc b/fastdeploy/vision/ocr/ppocr/ppocr_v2.cc index 06cb476029..7ad1f105ca 100644 --- a/fastdeploy/vision/ocr/ppocr/ppocr_v2.cc +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v2.cc @@ -110,7 +110,6 @@ bool PPOCRv2::Predict(cv::Mat* img, if (nullptr != classifier_ && result->cls_labels[i] % 2 == 1 && result->cls_scores[i] > classifier_->cls_thresh) { cv::rotate(image_list[i], image_list[i], 1); } - if (nullptr != recognizer_ && !Recognize(&(image_list[i]), result)) { FDERROR << "Failed to recgnize croped image of index " << i << "." << std::endl; return false; diff --git a/fastdeploy/vision/ocr/ppocr/recognizer.cc b/fastdeploy/vision/ocr/ppocr/recognizer.cc index e2a93e9a6a..4ca52df12d 100644 --- a/fastdeploy/vision/ocr/ppocr/recognizer.cc +++ b/fastdeploy/vision/ocr/ppocr/recognizer.cc @@ -48,7 +48,7 @@ Recognizer::Recognizer(const std::string& model_file, Backend::OPENVINO}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; } diff --git a/fastdeploy/vision/segmentation/ppseg/model.cc b/fastdeploy/vision/segmentation/ppseg/model.cc index 21c3774855..222f30d2fb 100755 --- a/fastdeploy/vision/segmentation/ppseg/model.cc +++ b/fastdeploy/vision/segmentation/ppseg/model.cc @@ -26,7 +26,7 @@ PaddleSegModel::PaddleSegModel(const std::string& model_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT}; + valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; @@ -106,7 +106,7 @@ bool PaddleSegModel::BuildPreprocessPipelineFromConfig() { << "Please refer to https://github.com/PaddlePaddle/PaddleSeg/blob/develop/docs/model_export.md" << " to export model with fixed input shape." << std::endl; - valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER}; + valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER}; } if (input_height != -1 && input_width != -1 && !yml_contain_resize_op) { diff --git a/java/android/.gitignore b/java/android/.gitignore new file mode 100644 index 0000000000..f6eba672f0 --- /dev/null +++ b/java/android/.gitignore @@ -0,0 +1,20 @@ +.DS_Store +.idea +.gradle +.cxx +cache +build +app/cache +app/libs/fastdeploy* +app/.cxx +app/build +app/src/main/assets/models/* +app/.gradle +app/.idea +fastdeploy/cache +fastdeploy/libs/fastdeploy* +fastdeploy/.cxx +fastdeploy/build +fastdeploy/src/main/assets/models/* +fastdeploy/.gradle +fastdeploy/.idea diff --git a/java/android/README.md b/java/android/README.md new file mode 100644 index 0000000000..42b6f9a714 --- /dev/null +++ b/java/android/README.md @@ -0,0 +1,3 @@ +# FastDeploy Android AAR 包使用文档 + +- TODO diff --git a/java/android/app/build.gradle b/java/android/app/build.gradle new file mode 100644 index 0000000000..d106ff18a4 --- /dev/null +++ b/java/android/app/build.gradle @@ -0,0 +1,82 @@ +import java.security.MessageDigest + +apply plugin: 'com.android.application' + +android { + compileSdk 28 + + defaultConfig { + applicationId 'com.baidu.paddle.fastdeploy.app.examples' + minSdkVersion 15 + //noinspection ExpiredTargetSdkVersion + targetSdkVersion 28 + versionCode 1 + versionName "1.0" + testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + +} + +dependencies { + implementation fileTree(include: ['*.jar'], dir: 'libs') + implementation 'com.android.support:appcompat-v7:28.0.0' + //noinspection GradleDependency + implementation 'com.android.support.constraint:constraint-layout:1.1.3' + implementation 'com.android.support:design:28.0.0' + implementation 'org.jetbrains:annotations:15.0' + implementation project(path: ':fastdeploy') + //noinspection GradleDependency + testImplementation 'junit:junit:4.12' + androidTestImplementation 'com.android.support.test:runner:1.0.2' + androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' +} + +def archives = [ + [ + 'src': 'https://bj.bcebos.com/paddlehub/fastdeploy/picodet_s_320_coco_lcnet.tgz', + 'dest' : 'src/main/assets/models' + ], + [ + 'src': 'https://bj.bcebos.com/paddlehub/fastdeploy/picodet_l_320_coco_lcnet.tgz', + 'dest' : 'src/main/assets/models' + ] +] + +task downloadAndExtractArchives(type: DefaultTask) { + doFirst { + println "Downloading and extracting archives including models ..." + } + doLast { + // Prepare cache folder for archives + String cachePath = "cache" + if (!file("${cachePath}").exists()) { + mkdir "${cachePath}" + } + archives.eachWithIndex { archive, index -> + MessageDigest messageDigest = MessageDigest.getInstance('MD5') + messageDigest.update(archive.src.bytes) + String cacheName = new BigInteger(1, messageDigest.digest()).toString(32) + // Download the target archive if not exists + boolean copyFiles = !file("${archive.dest}").exists() + if (!file("${cachePath}/${cacheName}.tgz").exists()) { + ant.get(src: archive.src, dest: file("${cachePath}/${cacheName}.tgz")) + copyFiles = true // force to copy files from the latest archive files + } + // Extract the target archive if its dest path does not exists + if (copyFiles) { + copy { + from tarTree("${cachePath}/${cacheName}.tgz") + into "${archive.dest}" + } + } + } + } +} +preBuild.dependsOn downloadAndExtractArchives \ No newline at end of file diff --git a/java/android/app/proguard-rules.pro b/java/android/app/proguard-rules.pro new file mode 100644 index 0000000000..481bb43481 --- /dev/null +++ b/java/android/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/java/android/app/src/androidTest/java/com/baidu/paddle/fastdeploy/ExampleInstrumentedTest.java b/java/android/app/src/androidTest/java/com/baidu/paddle/fastdeploy/ExampleInstrumentedTest.java new file mode 100644 index 0000000000..0efacb7909 --- /dev/null +++ b/java/android/app/src/androidTest/java/com/baidu/paddle/fastdeploy/ExampleInstrumentedTest.java @@ -0,0 +1,26 @@ +package com.baidu.paddle.fastdeploy; + +import android.content.Context; + +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.ext.junit.runners.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); + assertEquals("com.baidu.paddle.fastdeploy", appContext.getPackageName()); + } +} \ No newline at end of file diff --git a/java/android/app/src/main/AndroidManifest.xml b/java/android/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..9b00c8aacf --- /dev/null +++ b/java/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/assets/labels/coco_label_list.txt b/java/android/app/src/main/assets/labels/coco_label_list.txt new file mode 100644 index 0000000000..941cb4e139 --- /dev/null +++ b/java/android/app/src/main/assets/labels/coco_label_list.txt @@ -0,0 +1,80 @@ +person +bicycle +car +motorcycle +airplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +couch +potted plant +bed +dining table +toilet +tv +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/java/android/app/src/main/assets/labels/pascalvoc_label_list b/java/android/app/src/main/assets/labels/pascalvoc_label_list new file mode 100644 index 0000000000..87df23ce0a --- /dev/null +++ b/java/android/app/src/main/assets/labels/pascalvoc_label_list @@ -0,0 +1,21 @@ +background +aeroplane +bicycle +bird +boat +bottle +bus +car +cat +chair +cow +diningtable +dog +horse +motorbike +person +pottedplant +sheep +sofa +train +tvmonitor diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/MainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/MainActivity.java new file mode 100644 index 0000000000..cfc523256b --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/MainActivity.java @@ -0,0 +1,255 @@ +package com.baidu.paddle.fastdeploy.app.examples.detection; + + +import android.Manifest; +import android.annotation.SuppressLint; +import android.app.Activity; +import android.app.AlertDialog; +import android.content.DialogInterface; +import android.content.Intent; +import android.content.SharedPreferences; +import android.content.pm.PackageManager; +import android.graphics.Bitmap; +import android.os.Bundle; +import android.preference.PreferenceManager; +import android.support.annotation.NonNull; +import android.support.v4.app.ActivityCompat; +import android.support.v4.content.ContextCompat; +import android.view.View; +import android.view.Window; +import android.view.WindowManager; +import android.widget.ImageButton; +import android.widget.ImageView; +import android.widget.TextView; +import android.widget.Toast; + +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.app.examples.detection.R; +import com.baidu.paddle.fastdeploy.app.ui.CameraSurfaceView; +import com.baidu.paddle.fastdeploy.app.ui.Utils; +import com.baidu.paddle.fastdeploy.vision.DetectionResult; +import com.baidu.paddle.fastdeploy.vision.detection.PicoDet; + +import java.io.File; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class MainActivity extends Activity implements View.OnClickListener, CameraSurfaceView.OnTextureChangedListener { + private static final String TAG = MainActivity.class.getSimpleName(); + + CameraSurfaceView svPreview; + TextView tvStatus; + ImageButton btnSwitch; + ImageButton btnShutter; + ImageButton btnSettings; + ImageView realtimeToggleButton; + boolean isRealtimeStatusRunning = false; + ImageView backInPreview; + + String savedImagePath = "result.jpg"; + int lastFrameIndex = 0; + long lastFrameTime; + + // Call 'init' and 'release' manually later + PicoDet predictor = new PicoDet(); + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + // Fullscreen + requestWindowFeature(Window.FEATURE_NO_TITLE); + getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN); + + setContentView(R.layout.activity_main); + + // Clear all setting items to avoid app crashing due to the incorrect settings + initSettings(); + + // Init the camera preview and UI components + initView(); + + // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions + if (!checkAllPermissions()) { + requestAllPermissions(); + } + } + + @SuppressLint("NonConstantResourceId") + @Override + public void onClick(View v) { + switch (v.getId()) { + case R.id.btn_switch: + svPreview.switchCamera(); + break; + case R.id.btn_shutter: + @SuppressLint("SimpleDateFormat") + SimpleDateFormat date = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss"); + synchronized (this) { + savedImagePath = Utils.getDCIMDirectory() + File.separator + date.format(new Date()).toString() + ".png"; + } + Toast.makeText(MainActivity.this, "Save snapshot to " + savedImagePath, Toast.LENGTH_SHORT).show(); + break; + case R.id.btn_settings: + startActivity(new Intent(MainActivity.this, SettingsActivity.class)); + break; + case R.id.realtime_toggle_btn: + toggleRealtimeStyle(); + break; + case R.id.back_in_preview: + finish(); + break; + } + } + + private void toggleRealtimeStyle() { + if (isRealtimeStatusRunning) { + isRealtimeStatusRunning = false; + realtimeToggleButton.setImageResource(R.drawable.realtime_stop_btn); + svPreview.setOnTextureChangedListener(this); + tvStatus.setVisibility(View.VISIBLE); + } else { + isRealtimeStatusRunning = true; + realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); + tvStatus.setVisibility(View.GONE); + svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + return false; + } + }); + } + } + + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + String savedImagePath = ""; + synchronized (this) { + savedImagePath = MainActivity.this.savedImagePath; + } + boolean modified = false; + DetectionResult result = predictor.predict( + ARGB8888ImageBitmap, savedImagePath, SettingsActivity.scoreThreshold); + modified = result.initialized(); + if (!savedImagePath.isEmpty()) { + synchronized (this) { + MainActivity.this.savedImagePath = "result.jpg"; + } + } + lastFrameIndex++; + if (lastFrameIndex >= 30) { + final int fps = (int) (lastFrameIndex * 1e9 / (System.nanoTime() - lastFrameTime)); + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + tvStatus.setText(Integer.toString(fps) + "fps"); + } + }); + lastFrameIndex = 0; + lastFrameTime = System.nanoTime(); + } + return modified; + } + + @Override + protected void onResume() { + super.onResume(); + // Reload settings and re-initialize the predictor + checkAndUpdateSettings(); + // Open camera until the permissions have been granted + if (!checkAllPermissions()) { + svPreview.disableCamera(); + } + svPreview.onResume(); + } + + @Override + protected void onPause() { + super.onPause(); + svPreview.onPause(); + } + + @Override + protected void onDestroy() { + if (predictor != null) { + predictor.release(); + } + super.onDestroy(); + } + + public void initView() { + svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); + svPreview.setOnTextureChangedListener(this); + tvStatus = (TextView) findViewById(R.id.tv_status); + btnSwitch = (ImageButton) findViewById(R.id.btn_switch); + btnSwitch.setOnClickListener(this); + btnShutter = (ImageButton) findViewById(R.id.btn_shutter); + btnShutter.setOnClickListener(this); + btnSettings = (ImageButton) findViewById(R.id.btn_settings); + btnSettings.setOnClickListener(this); + realtimeToggleButton = findViewById(R.id.realtime_toggle_btn); + realtimeToggleButton.setOnClickListener(this); + backInPreview = findViewById(R.id.back_in_preview); + backInPreview.setOnClickListener(this); + } + + @SuppressLint("ApplySharedPref") + public void initSettings() { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.clear(); + editor.commit(); + SettingsActivity.resetSettings(); + } + + public void checkAndUpdateSettings() { + if (SettingsActivity.checkAndUpdateSettings(this)) { + String realModelDir = getCacheDir() + "/" + SettingsActivity.modelDir; + Utils.copyDirectoryFromAssets(this, SettingsActivity.modelDir, realModelDir); + String realLabelPath = getCacheDir() + "/" + SettingsActivity.labelPath; + Utils.copyFileFromAssets(this, SettingsActivity.labelPath, realLabelPath); + + String modelFile = realModelDir + "/" + "model.pdmodel"; + String paramsFile = realModelDir + "/" + "model.pdiparams"; + String configFile = realModelDir + "/" + "infer_cfg.yml"; + String labelFile = realLabelPath; + RuntimeOption option = new RuntimeOption(); + option.setCpuThreadNum(SettingsActivity.cpuThreadNum); + option.setLitePowerMode(SettingsActivity.cpuPowerMode); + option.enableRecordTimeOfRuntime(); + if (Boolean.parseBoolean(SettingsActivity.enableLiteFp16)) { + option.enableLiteFp16(); + } + predictor.init(modelFile, paramsFile, configFile, labelFile, option); + } + } + + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, + @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (grantResults[0] != PackageManager.PERMISSION_GRANTED || grantResults[1] != PackageManager.PERMISSION_GRANTED) { + new AlertDialog.Builder(MainActivity.this) + .setTitle("Permission denied") + .setMessage("Click to force quit the app, then open Settings->Apps & notifications->Target " + + "App->Permissions to grant all of the permissions.") + .setCancelable(false) + .setPositiveButton("Exit", new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + MainActivity.this.finish(); + } + }).show(); + } + } + + private void requestAllPermissions() { + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.CAMERA}, 0); + } + + private boolean checkAllPermissions() { + return ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED + && ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; + } +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/SettingsActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/SettingsActivity.java new file mode 100644 index 0000000000..4d4d875f33 --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/SettingsActivity.java @@ -0,0 +1,198 @@ +package com.baidu.paddle.fastdeploy.app.examples.detection; + +import android.annotation.SuppressLint; +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Bundle; +import android.preference.EditTextPreference; +import android.preference.ListPreference; +import android.preference.PreferenceManager; +import android.support.v7.app.ActionBar; + +import com.baidu.paddle.fastdeploy.app.examples.detection.R; +import com.baidu.paddle.fastdeploy.app.ui.AppCompatPreferenceActivity; +import com.baidu.paddle.fastdeploy.app.ui.Utils; + +import java.util.ArrayList; +import java.util.List; + +public class SettingsActivity extends AppCompatPreferenceActivity implements + SharedPreferences.OnSharedPreferenceChangeListener { + private static final String TAG = SettingsActivity.class.getSimpleName(); + + static public int selectedModelIdx = -1; + static public String modelDir = ""; + static public String labelPath = ""; + static public int cpuThreadNum = 2; + static public String cpuPowerMode = ""; + static public float scoreThreshold = 0.4f; + static public String enableLiteFp16 = "true"; + + ListPreference lpChoosePreInstalledModel = null; + EditTextPreference etModelDir = null; + EditTextPreference etLabelPath = null; + ListPreference lpCPUThreadNum = null; + ListPreference lpCPUPowerMode = null; + EditTextPreference etScoreThreshold = null; + ListPreference lpEnableLiteFp16 = null; + + List preInstalledModelDirs = null; + List preInstalledLabelPaths = null; + List preInstalledCPUThreadNums = null; + List preInstalledCPUPowerModes = null; + List preInstalledScoreThresholds = null; + List preInstalledEnableLiteFp16s = null; + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + addPreferencesFromResource(R.xml.settings); + ActionBar supportActionBar = getSupportActionBar(); + if (supportActionBar != null) { + supportActionBar.setDisplayHomeAsUpEnabled(true); + } + + // Initialize pre-installed models + preInstalledModelDirs = new ArrayList(); + preInstalledLabelPaths = new ArrayList(); + preInstalledCPUThreadNums = new ArrayList(); + preInstalledCPUPowerModes = new ArrayList(); + preInstalledScoreThresholds = new ArrayList(); + preInstalledEnableLiteFp16s = new ArrayList(); + preInstalledModelDirs.add(getString(R.string.MODEL_DIR_DEFAULT)); + preInstalledLabelPaths.add(getString(R.string.LABEL_PATH_DEFAULT)); + preInstalledCPUThreadNums.add(getString(R.string.CPU_THREAD_NUM_DEFAULT)); + preInstalledCPUPowerModes.add(getString(R.string.CPU_POWER_MODE_DEFAULT)); + preInstalledScoreThresholds.add(getString(R.string.SCORE_THRESHOLD_DEFAULT)); + preInstalledEnableLiteFp16s.add(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + // Setup UI components + lpChoosePreInstalledModel = + (ListPreference) findPreference(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY)); + String[] preInstalledModelNames = new String[preInstalledModelDirs.size()]; + for (int i = 0; i < preInstalledModelDirs.size(); i++) { + preInstalledModelNames[i] = preInstalledModelDirs.get(i).substring(preInstalledModelDirs.get(i).lastIndexOf("/") + 1); + } + lpChoosePreInstalledModel.setEntries(preInstalledModelNames); + lpChoosePreInstalledModel.setEntryValues(preInstalledModelDirs.toArray(new String[preInstalledModelDirs.size()])); + lpCPUThreadNum = (ListPreference) findPreference(getString(R.string.CPU_THREAD_NUM_KEY)); + lpCPUPowerMode = (ListPreference) findPreference(getString(R.string.CPU_POWER_MODE_KEY)); + etModelDir = (EditTextPreference) findPreference(getString(R.string.MODEL_DIR_KEY)); + etModelDir.setTitle("Model dir (SDCard: " + Utils.getSDCardDirectory() + ")"); + etLabelPath = (EditTextPreference) findPreference(getString(R.string.LABEL_PATH_KEY)); + etLabelPath.setTitle("Label path (SDCard: " + Utils.getSDCardDirectory() + ")"); + etScoreThreshold = (EditTextPreference) findPreference(getString(R.string.SCORE_THRESHOLD_KEY)); + lpEnableLiteFp16 = (ListPreference) findPreference(getString(R.string.ENABLE_LITE_FP16_MODE_KEY)); + } + + @SuppressLint("ApplySharedPref") + private void reloadSettingsAndUpdateUI() { + SharedPreferences sharedPreferences = getPreferenceScreen().getSharedPreferences(); + + String selected_model_dir = sharedPreferences.getString(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY), + getString(R.string.MODEL_DIR_DEFAULT)); + int selected_model_idx = lpChoosePreInstalledModel.findIndexOfValue(selected_model_dir); + if (selected_model_idx >= 0 && selected_model_idx < preInstalledModelDirs.size() && selected_model_idx != selectedModelIdx) { + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.putString(getString(R.string.MODEL_DIR_KEY), preInstalledModelDirs.get(selected_model_idx)); + editor.putString(getString(R.string.LABEL_PATH_KEY), preInstalledLabelPaths.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_THREAD_NUM_KEY), preInstalledCPUThreadNums.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_POWER_MODE_KEY), preInstalledCPUPowerModes.get(selected_model_idx)); + editor.putString(getString(R.string.SCORE_THRESHOLD_KEY), preInstalledScoreThresholds.get(selected_model_idx)); + editor.putString(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT), preInstalledEnableLiteFp16s.get(selected_model_idx)); + editor.commit(); + lpChoosePreInstalledModel.setSummary(selected_model_dir); + selectedModelIdx = selected_model_idx; + } + + String model_dir = sharedPreferences.getString(getString(R.string.MODEL_DIR_KEY), + getString(R.string.MODEL_DIR_DEFAULT)); + String label_path = sharedPreferences.getString(getString(R.string.LABEL_PATH_KEY), + getString(R.string.LABEL_PATH_DEFAULT)); + String cpu_thread_num = sharedPreferences.getString(getString(R.string.CPU_THREAD_NUM_KEY), + getString(R.string.CPU_THREAD_NUM_DEFAULT)); + String cpu_power_mode = sharedPreferences.getString(getString(R.string.CPU_POWER_MODE_KEY), + getString(R.string.CPU_POWER_MODE_DEFAULT)); + String score_threshold = sharedPreferences.getString(getString(R.string.SCORE_THRESHOLD_KEY), + getString(R.string.SCORE_THRESHOLD_DEFAULT)); + String enable_lite_fp16 = sharedPreferences.getString(getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + etModelDir.setSummary(model_dir); + etLabelPath.setSummary(label_path); + lpCPUThreadNum.setValue(cpu_thread_num); + lpCPUThreadNum.setSummary(cpu_thread_num); + lpCPUPowerMode.setValue(cpu_power_mode); + lpCPUPowerMode.setSummary(cpu_power_mode); + etScoreThreshold.setSummary(score_threshold); + etScoreThreshold.setText(score_threshold); + lpEnableLiteFp16.setValue(enable_lite_fp16); + lpEnableLiteFp16.setSummary(enable_lite_fp16); + + } + + static boolean checkAndUpdateSettings(Context ctx) { + boolean settingsChanged = false; + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(ctx); + + String model_dir = sharedPreferences.getString(ctx.getString(R.string.MODEL_DIR_KEY), + ctx.getString(R.string.MODEL_DIR_DEFAULT)); + settingsChanged |= !modelDir.equalsIgnoreCase(model_dir); + modelDir = model_dir; + + String label_path = sharedPreferences.getString(ctx.getString(R.string.LABEL_PATH_KEY), + ctx.getString(R.string.LABEL_PATH_DEFAULT)); + settingsChanged |= !labelPath.equalsIgnoreCase(label_path); + labelPath = label_path; + + String cpu_thread_num = sharedPreferences.getString(ctx.getString(R.string.CPU_THREAD_NUM_KEY), + ctx.getString(R.string.CPU_THREAD_NUM_DEFAULT)); + settingsChanged |= cpuThreadNum != Integer.parseInt(cpu_thread_num); + cpuThreadNum = Integer.parseInt(cpu_thread_num); + + String cpu_power_mode = sharedPreferences.getString(ctx.getString(R.string.CPU_POWER_MODE_KEY), + ctx.getString(R.string.CPU_POWER_MODE_DEFAULT)); + settingsChanged |= !cpuPowerMode.equalsIgnoreCase(cpu_power_mode); + cpuPowerMode = cpu_power_mode; + + String score_threshold = sharedPreferences.getString(ctx.getString(R.string.SCORE_THRESHOLD_KEY), + ctx.getString(R.string.SCORE_THRESHOLD_DEFAULT)); + settingsChanged |= scoreThreshold != Float.parseFloat(score_threshold); + scoreThreshold = Float.parseFloat(score_threshold); + + String enable_lite_fp16 = sharedPreferences.getString(ctx.getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + ctx.getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + settingsChanged |= !enableLiteFp16.equalsIgnoreCase(enable_lite_fp16); + enableLiteFp16 = enable_lite_fp16; + + return settingsChanged; + } + + static void resetSettings() { + selectedModelIdx = -1; + modelDir = ""; + labelPath = ""; + cpuThreadNum = 2; + cpuPowerMode = ""; + scoreThreshold = 0.4f; + enableLiteFp16 = "true"; + } + + @Override + protected void onResume() { + super.onResume(); + getPreferenceScreen().getSharedPreferences().registerOnSharedPreferenceChangeListener(this); + reloadSettingsAndUpdateUI(); + } + + @Override + protected void onPause() { + super.onPause(); + getPreferenceScreen().getSharedPreferences().unregisterOnSharedPreferenceChangeListener(this); + } + + @Override + public void onSharedPreferenceChanged(SharedPreferences sharedPreferences, String key) { + reloadSettingsAndUpdateUI(); + } +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/ActionBarLayout.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/ActionBarLayout.java new file mode 100644 index 0000000000..6616a290b3 --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/ActionBarLayout.java @@ -0,0 +1,33 @@ +package com.baidu.paddle.fastdeploy.app.ui; + +import android.content.Context; +import android.graphics.Color; +import android.support.annotation.Nullable; +import android.util.AttributeSet; +import android.widget.RelativeLayout; + + +public class ActionBarLayout extends RelativeLayout { + private int layoutHeight = 150; + + public ActionBarLayout(Context context) { + super(context); + } + + public ActionBarLayout(Context context, @Nullable AttributeSet attrs) { + super(context, attrs); + } + + public ActionBarLayout(Context context, @Nullable AttributeSet attrs, int defStyleAttr) { + super(context, attrs, defStyleAttr); + } + + @Override + protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { + super.onMeasure(widthMeasureSpec, heightMeasureSpec); + int width = MeasureSpec.getSize(widthMeasureSpec); + setMeasuredDimension(width, layoutHeight); + setBackgroundColor(Color.BLACK); + setAlpha(0.9f); + } +} \ No newline at end of file diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/AppCompatPreferenceActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/AppCompatPreferenceActivity.java new file mode 100644 index 0000000000..8195c52fa8 --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/AppCompatPreferenceActivity.java @@ -0,0 +1,111 @@ +package com.baidu.paddle.fastdeploy.app.ui; + +import android.content.res.Configuration; +import android.os.Bundle; +import android.preference.PreferenceActivity; +import android.support.annotation.LayoutRes; +import android.support.annotation.Nullable; +import android.support.v7.app.ActionBar; +import android.support.v7.app.AppCompatDelegate; +import android.support.v7.widget.Toolbar; +import android.view.MenuInflater; +import android.view.View; +import android.view.ViewGroup; + +/** + * A {@link PreferenceActivity} which implements and proxies the necessary calls + * to be used with AppCompat. + *

+ * This technique can be used with an {@link android.app.Activity} class, not just + * {@link PreferenceActivity}. + */ +public abstract class AppCompatPreferenceActivity extends PreferenceActivity { + private AppCompatDelegate mDelegate; + + @Override + protected void onCreate(Bundle savedInstanceState) { + getDelegate().installViewFactory(); + getDelegate().onCreate(savedInstanceState); + super.onCreate(savedInstanceState); + } + + @Override + protected void onPostCreate(Bundle savedInstanceState) { + super.onPostCreate(savedInstanceState); + getDelegate().onPostCreate(savedInstanceState); + } + + public ActionBar getSupportActionBar() { + return getDelegate().getSupportActionBar(); + } + + public void setSupportActionBar(@Nullable Toolbar toolbar) { + getDelegate().setSupportActionBar(toolbar); + } + + @Override + public MenuInflater getMenuInflater() { + return getDelegate().getMenuInflater(); + } + + @Override + public void setContentView(@LayoutRes int layoutResID) { + getDelegate().setContentView(layoutResID); + } + + @Override + public void setContentView(View view) { + getDelegate().setContentView(view); + } + + @Override + public void setContentView(View view, ViewGroup.LayoutParams params) { + getDelegate().setContentView(view, params); + } + + @Override + public void addContentView(View view, ViewGroup.LayoutParams params) { + getDelegate().addContentView(view, params); + } + + @Override + protected void onPostResume() { + super.onPostResume(); + getDelegate().onPostResume(); + } + + @Override + protected void onTitleChanged(CharSequence title, int color) { + super.onTitleChanged(title, color); + getDelegate().setTitle(title); + } + + @Override + public void onConfigurationChanged(Configuration newConfig) { + super.onConfigurationChanged(newConfig); + getDelegate().onConfigurationChanged(newConfig); + } + + @Override + protected void onStop() { + super.onStop(); + getDelegate().onStop(); + } + + @Override + protected void onDestroy() { + super.onDestroy(); + getDelegate().onDestroy(); + } + + public void invalidateOptionsMenu() { + getDelegate().invalidateOptionsMenu(); + } + + private AppCompatDelegate getDelegate() { + if (mDelegate == null) { + mDelegate = AppCompatDelegate.create(this, null); + } + return mDelegate; + } +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/CameraSurfaceView.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/CameraSurfaceView.java new file mode 100644 index 0000000000..6db97d68a3 --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/CameraSurfaceView.java @@ -0,0 +1,329 @@ +package com.baidu.paddle.fastdeploy.app.ui; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.SurfaceTexture; +import android.hardware.Camera; +import android.hardware.Camera.CameraInfo; +import android.hardware.Camera.Size; +import android.opengl.GLES11Ext; +import android.opengl.GLES20; +import android.opengl.GLSurfaceView; +import android.opengl.GLSurfaceView.Renderer; +import android.opengl.GLUtils; +import android.opengl.Matrix; +import android.util.AttributeSet; +import android.util.Log; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; +import java.util.List; + +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.opengles.GL10; + +public class CameraSurfaceView extends GLSurfaceView implements Renderer, + SurfaceTexture.OnFrameAvailableListener { + private static final String TAG = CameraSurfaceView.class.getSimpleName(); + + public static final int EXPECTED_PREVIEW_WIDTH = 1280; + public static final int EXPECTED_PREVIEW_HEIGHT = 720; + + + protected int numberOfCameras; + protected int selectedCameraId; + protected boolean disableCamera = false; + protected Camera camera; + + protected Context context; + protected SurfaceTexture surfaceTexture; + protected int surfaceWidth = 0; + protected int surfaceHeight = 0; + protected int textureWidth = 0; + protected int textureHeight = 0; + + // In order to manipulate the camera preview data and render the modified one + // to the screen, three textures are created and the data flow is shown as following: + // previewdata->camTextureId->fboTexureId->drawTexureId->framebuffer + protected int[] fbo = {0}; + protected int[] camTextureId = {0}; + protected int[] fboTexureId = {0}; + protected int[] drawTexureId = {0}; + + private final String vss = "" + + "attribute vec2 vPosition;\n" + + "attribute vec2 vTexCoord;\n" + "varying vec2 texCoord;\n" + + "void main() {\n" + " texCoord = vTexCoord;\n" + + " gl_Position = vec4 (vPosition.x, vPosition.y, 0.0, 1.0);\n" + + "}"; + + private final String fssCam2FBO = "" + + "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + + "uniform samplerExternalOES sTexture;\n" + + "varying vec2 texCoord;\n" + + "void main() {\n" + + " gl_FragColor = texture2D(sTexture,texCoord);\n" + "}"; + + private final String fssTex2Screen = "" + + "precision mediump float;\n" + + "uniform sampler2D sTexture;\n" + + "varying vec2 texCoord;\n" + + "void main() {\n" + + " gl_FragColor = texture2D(sTexture,texCoord);\n" + "}"; + + private final float[] vertexCoords = { + -1, -1, + -1, 1, + 1, -1, + 1, 1}; + private float[] textureCoords = { + 0, 1, + 0, 0, + 1, 1, + 1, 0}; + + private FloatBuffer vertexCoordsBuffer; + private FloatBuffer textureCoordsBuffer; + + private int progCam2FBO = -1; + private int progTex2Screen = -1; + private int vcCam2FBO; + private int tcCam2FBO; + private int vcTex2Screen; + private int tcTex2Screen; + + public interface OnTextureChangedListener { + boolean onTextureChanged(Bitmap ARGB8888ImageBitmap); + } + + private OnTextureChangedListener onTextureChangedListener = null; + + public void setOnTextureChangedListener(OnTextureChangedListener listener) { + onTextureChangedListener = listener; + } + + public CameraSurfaceView(Context ctx, AttributeSet attrs) { + super(ctx, attrs); + context = ctx; + setEGLContextClientVersion(2); + setRenderer(this); + setRenderMode(RENDERMODE_WHEN_DIRTY); + + // Find the total number of available cameras and the ID of the default camera + numberOfCameras = Camera.getNumberOfCameras(); + CameraInfo cameraInfo = new CameraInfo(); + for (int i = 0; i < numberOfCameras; i++) { + Camera.getCameraInfo(i, cameraInfo); + if (cameraInfo.facing == CameraInfo.CAMERA_FACING_BACK) { + selectedCameraId = i; + } + } + } + + @Override + public void onSurfaceCreated(GL10 gl, EGLConfig config) { + // Create OES texture for storing camera preview data(YUV format) + GLES20.glGenTextures(1, camTextureId, 0); + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, camTextureId[0]); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST); + surfaceTexture = new SurfaceTexture(camTextureId[0]); + surfaceTexture.setOnFrameAvailableListener(this); + + // Prepare vertex and texture coordinates + int bytes = vertexCoords.length * Float.SIZE / Byte.SIZE; + vertexCoordsBuffer = ByteBuffer.allocateDirect(bytes).order(ByteOrder.nativeOrder()).asFloatBuffer(); + textureCoordsBuffer = ByteBuffer.allocateDirect(bytes).order(ByteOrder.nativeOrder()).asFloatBuffer(); + vertexCoordsBuffer.put(vertexCoords).position(0); + textureCoordsBuffer.put(textureCoords).position(0); + + // Create vertex and fragment shaders + // camTextureId->fboTexureId + progCam2FBO = Utils.createShaderProgram(vss, fssCam2FBO); + vcCam2FBO = GLES20.glGetAttribLocation(progCam2FBO, "vPosition"); + tcCam2FBO = GLES20.glGetAttribLocation(progCam2FBO, "vTexCoord"); + GLES20.glEnableVertexAttribArray(vcCam2FBO); + GLES20.glEnableVertexAttribArray(tcCam2FBO); + // fboTexureId/drawTexureId -> screen + progTex2Screen = Utils.createShaderProgram(vss, fssTex2Screen); + vcTex2Screen = GLES20.glGetAttribLocation(progTex2Screen, "vPosition"); + tcTex2Screen = GLES20.glGetAttribLocation(progTex2Screen, "vTexCoord"); + GLES20.glEnableVertexAttribArray(vcTex2Screen); + GLES20.glEnableVertexAttribArray(tcTex2Screen); + } + + @Override + public void onSurfaceChanged(GL10 gl, int width, int height) { + surfaceWidth = width; + surfaceHeight = height; + openCamera(); + } + + @Override + public void onDrawFrame(GL10 gl) { + if (surfaceTexture == null) return; + + GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT); + surfaceTexture.updateTexImage(); + float[] matrix = new float[16]; + surfaceTexture.getTransformMatrix(matrix); + + // camTextureId->fboTexureId + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fbo[0]); + GLES20.glViewport(0, 0, textureWidth, textureHeight); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + GLES20.glUseProgram(progCam2FBO); + GLES20.glVertexAttribPointer(vcCam2FBO, 2, GLES20.GL_FLOAT, false, 4 * 2, vertexCoordsBuffer); + textureCoordsBuffer.clear(); + textureCoordsBuffer.put(transformTextureCoordinates(textureCoords, matrix)); + textureCoordsBuffer.position(0); + GLES20.glVertexAttribPointer(tcCam2FBO, 2, GLES20.GL_FLOAT, false, 4 * 2, textureCoordsBuffer); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, camTextureId[0]); + GLES20.glUniform1i(GLES20.glGetUniformLocation(progCam2FBO, "sTexture"), 0); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + GLES20.glFlush(); + + // Check if the draw texture is set + int targetTexureId = fboTexureId[0]; + if (onTextureChangedListener != null) { + // Read pixels of FBO to a bitmap + ByteBuffer pixelBuffer = ByteBuffer.allocate(textureWidth * textureHeight * 4); + GLES20.glReadPixels(0, 0, textureWidth, textureHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuffer); + Bitmap ARGB8888ImageBitmap = Bitmap.createBitmap(textureWidth, textureHeight, Bitmap.Config.ARGB_8888); + ARGB8888ImageBitmap.copyPixelsFromBuffer(pixelBuffer); + boolean modified = onTextureChangedListener.onTextureChanged(ARGB8888ImageBitmap); + if (modified) { + targetTexureId = drawTexureId[0]; + // Update a bitmap to the GL texture if modified + GLES20.glActiveTexture(targetTexureId); + // GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, targetTexureId); + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, targetTexureId); + GLUtils.texImage2D(GL10.GL_TEXTURE_2D, 0, ARGB8888ImageBitmap, 0); + } + ARGB8888ImageBitmap.recycle(); + } + + // fboTexureId/drawTexureId->Screen + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + GLES20.glUseProgram(progTex2Screen); + GLES20.glVertexAttribPointer(vcTex2Screen, 2, GLES20.GL_FLOAT, false, 4 * 2, vertexCoordsBuffer); + textureCoordsBuffer.clear(); + textureCoordsBuffer.put(textureCoords); + textureCoordsBuffer.position(0); + GLES20.glVertexAttribPointer(tcTex2Screen, 2, GLES20.GL_FLOAT, false, 4 * 2, textureCoordsBuffer); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, targetTexureId); + GLES20.glUniform1i(GLES20.glGetUniformLocation(progTex2Screen, "sTexture"), 0); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + GLES20.glFlush(); + } + + private float[] transformTextureCoordinates(float[] coords, float[] matrix) { + float[] result = new float[coords.length]; + float[] vt = new float[4]; + for (int i = 0; i < coords.length; i += 2) { + float[] v = {coords[i], coords[i + 1], 0, 1}; + Matrix.multiplyMV(vt, 0, matrix, 0, v, 0); + result[i] = vt[0]; + result[i + 1] = vt[1]; + } + return result; + } + + @Override + public void onResume() { + super.onResume(); + } + + @Override + public void onPause() { + super.onPause(); + releaseCamera(); + } + + @Override + public void onFrameAvailable(SurfaceTexture surfaceTexture) { + requestRender(); + } + + public void disableCamera() { + disableCamera = true; + } + + public void switchCamera() { + releaseCamera(); + selectedCameraId = (selectedCameraId + 1) % numberOfCameras; + openCamera(); + } + + public void openCamera() { + if (disableCamera) return; + camera = Camera.open(selectedCameraId); + List supportedPreviewSizes = camera.getParameters().getSupportedPreviewSizes(); + Size previewSize = Utils.getOptimalPreviewSize(supportedPreviewSizes, EXPECTED_PREVIEW_WIDTH, + EXPECTED_PREVIEW_HEIGHT); + Camera.Parameters parameters = camera.getParameters(); + parameters.setPreviewSize(previewSize.width, previewSize.height); + if (parameters.getSupportedFocusModes().contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) { + parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); + } + camera.setParameters(parameters); + int degree = Utils.getCameraDisplayOrientation(context, selectedCameraId); + camera.setDisplayOrientation(degree); + boolean rotate = degree == 90 || degree == 270; + textureWidth = rotate ? previewSize.height : previewSize.width; + textureHeight = rotate ? previewSize.width : previewSize.height; + // Destroy FBO and draw textures + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + GLES20.glDeleteFramebuffers(1, fbo, 0); + GLES20.glDeleteTextures(1, drawTexureId, 0); + GLES20.glDeleteTextures(1, fboTexureId, 0); + // Normal texture for storing modified camera preview data(RGBA format) + GLES20.glGenTextures(1, drawTexureId, 0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, drawTexureId[0]); + GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, textureWidth, textureHeight, 0, + GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST); + // FBO texture for storing camera preview data(RGBA format) + GLES20.glGenTextures(1, fboTexureId, 0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, fboTexureId[0]); + GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, textureWidth, textureHeight, 0, + GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST); + // Generate FBO and bind to FBO texture + GLES20.glGenFramebuffers(1, fbo, 0); + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fbo[0]); + GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, + fboTexureId[0], 0); + try { + camera.setPreviewTexture(surfaceTexture); + } catch (IOException exception) { + Log.e(TAG, "IOException caused by setPreviewDisplay()", exception); + } + camera.startPreview(); + } + + public void releaseCamera() { + if (camera != null) { + camera.setPreviewCallback(null); + camera.stopPreview(); + camera.release(); + camera = null; + } + } +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java new file mode 100644 index 0000000000..ef990237fd --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java @@ -0,0 +1,244 @@ +package com.baidu.paddle.fastdeploy.app.ui; + +import android.content.Context; +import android.content.res.Resources; +import android.hardware.Camera; +import android.opengl.GLES20; +import android.os.Environment; +import android.util.Log; +import android.view.Surface; +import android.view.WindowManager; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; + +public class Utils { + private static final String TAG = Utils.class.getSimpleName(); + + public static void RecursiveCreateDirectories(String fileDir) { + String[] fileDirs = fileDir.split("\\/"); + String topPath = ""; + for (int i = 0; i < fileDirs.length; i++) { + topPath += "/" + fileDirs[i]; + File file = new File(topPath); + if (file.exists()) { + continue; + } else { + file.mkdir(); + } + } + } + + public static void copyFileFromAssets(Context appCtx, String srcPath, String dstPath) { + if (srcPath.isEmpty() || dstPath.isEmpty()) { + return; + } + String dstDir = dstPath.substring(0, dstPath.lastIndexOf('/')); + if (dstDir.length() > 0) { + RecursiveCreateDirectories(dstDir); + } + InputStream is = null; + OutputStream os = null; + try { + is = new BufferedInputStream(appCtx.getAssets().open(srcPath)); + os = new BufferedOutputStream(new FileOutputStream(new File(dstPath))); + byte[] buffer = new byte[1024]; + int length = 0; + while ((length = is.read(buffer)) != -1) { + os.write(buffer, 0, length); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + os.close(); + is.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + public static void copyDirectoryFromAssets(Context appCtx, String srcDir, String dstDir) { + if (srcDir.isEmpty() || dstDir.isEmpty()) { + return; + } + try { + if (!new File(dstDir).exists()) { + new File(dstDir).mkdirs(); + } + for (String fileName : appCtx.getAssets().list(srcDir)) { + String srcSubPath = srcDir + File.separator + fileName; + String dstSubPath = dstDir + File.separator + fileName; + if (new File(srcSubPath).isDirectory()) { + copyDirectoryFromAssets(appCtx, srcSubPath, dstSubPath); + } else { + copyFileFromAssets(appCtx, srcSubPath, dstSubPath); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static float[] parseFloatsFromString(String string, String delimiter) { + String[] pieces = string.trim().toLowerCase().split(delimiter); + float[] floats = new float[pieces.length]; + for (int i = 0; i < pieces.length; i++) { + floats[i] = Float.parseFloat(pieces[i].trim()); + } + return floats; + } + + public static long[] parseLongsFromString(String string, String delimiter) { + String[] pieces = string.trim().toLowerCase().split(delimiter); + long[] longs = new long[pieces.length]; + for (int i = 0; i < pieces.length; i++) { + longs[i] = Long.parseLong(pieces[i].trim()); + } + return longs; + } + + public static String getSDCardDirectory() { + return Environment.getExternalStorageDirectory().getAbsolutePath(); + } + + public static String getDCIMDirectory() { + return Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DCIM).getAbsolutePath(); + } + + public static Camera.Size getOptimalPreviewSize(List sizes, int w, int h) { + final double ASPECT_TOLERANCE = 0.1; + double targetRatio = (double) w / h; + if (sizes == null) return null; + + Camera.Size optimalSize = null; + double minDiff = Double.MAX_VALUE; + + int targetHeight = h; + + // Try to find an size match aspect ratio and size + for (Camera.Size size : sizes) { + double ratio = (double) size.width / size.height; + if (Math.abs(ratio - targetRatio) > ASPECT_TOLERANCE) continue; + if (Math.abs(size.height - targetHeight) < minDiff) { + optimalSize = size; + minDiff = Math.abs(size.height - targetHeight); + } + } + + // Cannot find the one match the aspect ratio, ignore the requirement + if (optimalSize == null) { + minDiff = Double.MAX_VALUE; + for (Camera.Size size : sizes) { + if (Math.abs(size.height - targetHeight) < minDiff) { + optimalSize = size; + minDiff = Math.abs(size.height - targetHeight); + } + } + } + return optimalSize; + } + + public static int getScreenWidth() { + return Resources.getSystem().getDisplayMetrics().widthPixels; + } + + public static int getScreenHeight() { + return Resources.getSystem().getDisplayMetrics().heightPixels; + } + + public static int getCameraDisplayOrientation(Context context, int cameraId) { + Camera.CameraInfo info = new Camera.CameraInfo(); + Camera.getCameraInfo(cameraId, info); + WindowManager wm = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); + int rotation = wm.getDefaultDisplay().getRotation(); + int degrees = 0; + switch (rotation) { + case Surface.ROTATION_0: + degrees = 0; + break; + case Surface.ROTATION_90: + degrees = 90; + break; + case Surface.ROTATION_180: + degrees = 180; + break; + case Surface.ROTATION_270: + degrees = 270; + break; + } + int result; + if (info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) { + result = (info.orientation + degrees) % 360; + result = (360 - result) % 360; // compensate the mirror + } else { + // back-facing + result = (info.orientation - degrees + 360) % 360; + } + return result; + } + + public static int createShaderProgram(String vss, String fss) { + int vshader = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER); + GLES20.glShaderSource(vshader, vss); + GLES20.glCompileShader(vshader); + int[] status = new int[1]; + GLES20.glGetShaderiv(vshader, GLES20.GL_COMPILE_STATUS, status, 0); + if (status[0] == 0) { + Log.e(TAG, GLES20.glGetShaderInfoLog(vshader)); + GLES20.glDeleteShader(vshader); + vshader = 0; + return 0; + } + + int fshader = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER); + GLES20.glShaderSource(fshader, fss); + GLES20.glCompileShader(fshader); + GLES20.glGetShaderiv(fshader, GLES20.GL_COMPILE_STATUS, status, 0); + if (status[0] == 0) { + Log.e(TAG, GLES20.glGetShaderInfoLog(fshader)); + GLES20.glDeleteShader(vshader); + GLES20.glDeleteShader(fshader); + fshader = 0; + return 0; + } + + int program = GLES20.glCreateProgram(); + GLES20.glAttachShader(program, vshader); + GLES20.glAttachShader(program, fshader); + GLES20.glLinkProgram(program); + GLES20.glDeleteShader(vshader); + GLES20.glDeleteShader(fshader); + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, status, 0); + if (status[0] == 0) { + Log.e(TAG, GLES20.glGetProgramInfoLog(program)); + program = 0; + return 0; + } + GLES20.glValidateProgram(program); + GLES20.glGetProgramiv(program, GLES20.GL_VALIDATE_STATUS, status, 0); + if (status[0] == 0) { + Log.e(TAG, GLES20.glGetProgramInfoLog(program)); + GLES20.glDeleteProgram(program); + program = 0; + return 0; + } + + return program; + } + + public static boolean isSupportedNPU() { + String hardware = android.os.Build.HARDWARE; + return hardware.equalsIgnoreCase("kirin810") || hardware.equalsIgnoreCase("kirin990"); + } +} diff --git a/java/android/app/src/main/res/drawable-v24/action_button_layer.xml b/java/android/app/src/main/res/drawable-v24/action_button_layer.xml new file mode 100644 index 0000000000..a0d2e76bfa --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/action_button_layer.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/drawable-v24/album_btn.xml b/java/android/app/src/main/res/drawable-v24/album_btn.xml new file mode 100644 index 0000000000..26d01c5841 --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/album_btn.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/java/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 0000000000..1f6bb29060 --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/java/android/app/src/main/res/drawable-v24/realtime_start_btn.xml b/java/android/app/src/main/res/drawable-v24/realtime_start_btn.xml new file mode 100644 index 0000000000..6641344530 --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/realtime_start_btn.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/drawable-v24/realtime_stop_btn.xml b/java/android/app/src/main/res/drawable-v24/realtime_stop_btn.xml new file mode 100644 index 0000000000..8869a1b2bf --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/realtime_stop_btn.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/drawable-v24/result_page_border_section_bk.xml b/java/android/app/src/main/res/drawable-v24/result_page_border_section_bk.xml new file mode 100644 index 0000000000..bd068f169f --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/result_page_border_section_bk.xml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/drawable-v24/round_corner_btn.xml b/java/android/app/src/main/res/drawable-v24/round_corner_btn.xml new file mode 100644 index 0000000000..c5dcc45d56 --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/round_corner_btn.xml @@ -0,0 +1,10 @@ + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/drawable-v24/seekbar_progress_realtime.xml b/java/android/app/src/main/res/drawable-v24/seekbar_progress_realtime.xml new file mode 100644 index 0000000000..b349d15a6a --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/seekbar_progress_realtime.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + diff --git a/java/android/app/src/main/res/drawable-v24/seekbar_progress_result.xml b/java/android/app/src/main/res/drawable-v24/seekbar_progress_result.xml new file mode 100644 index 0000000000..17cb68ed80 --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/seekbar_progress_result.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/java/android/app/src/main/res/drawable-v24/seekbar_thumb.xml b/java/android/app/src/main/res/drawable-v24/seekbar_thumb.xml new file mode 100644 index 0000000000..96bd95e0a1 --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/seekbar_thumb.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/drawable-v24/seekbar_thumb_shape.xml b/java/android/app/src/main/res/drawable-v24/seekbar_thumb_shape.xml new file mode 100644 index 0000000000..26d033b6df --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/seekbar_thumb_shape.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + diff --git a/java/android/app/src/main/res/drawable-v24/switch_side_btn.xml b/java/android/app/src/main/res/drawable-v24/switch_side_btn.xml new file mode 100644 index 0000000000..b9b2edfb6a --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/switch_side_btn.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/drawable-v24/take_picture_btn.xml b/java/android/app/src/main/res/drawable-v24/take_picture_btn.xml new file mode 100644 index 0000000000..4966675c35 --- /dev/null +++ b/java/android/app/src/main/res/drawable-v24/take_picture_btn.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/drawable-xhdpi/album.png b/java/android/app/src/main/res/drawable-xhdpi/album.png new file mode 100644 index 0000000000..3a6fdedaee Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/album.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/album_pressed.png b/java/android/app/src/main/res/drawable-xhdpi/album_pressed.png new file mode 100644 index 0000000000..aa873424eb Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/album_pressed.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/back_btn.png b/java/android/app/src/main/res/drawable-xhdpi/back_btn.png new file mode 100644 index 0000000000..ff121e85f5 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/back_btn.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/more_menu.png b/java/android/app/src/main/res/drawable-xhdpi/more_menu.png new file mode 100644 index 0000000000..edf9f3ccce Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/more_menu.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/realtime_start.png b/java/android/app/src/main/res/drawable-xhdpi/realtime_start.png new file mode 100644 index 0000000000..94ab081724 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/realtime_start.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/realtime_start_pressed.png b/java/android/app/src/main/res/drawable-xhdpi/realtime_start_pressed.png new file mode 100644 index 0000000000..feef0fea62 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/realtime_start_pressed.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/realtime_stop.png b/java/android/app/src/main/res/drawable-xhdpi/realtime_stop.png new file mode 100644 index 0000000000..8c926367db Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/realtime_stop.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/realtime_stop_pressed.png b/java/android/app/src/main/res/drawable-xhdpi/realtime_stop_pressed.png new file mode 100644 index 0000000000..309082788b Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/realtime_stop_pressed.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/scan_icon.png b/java/android/app/src/main/res/drawable-xhdpi/scan_icon.png new file mode 100644 index 0000000000..7517d99d09 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/scan_icon.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/seekbar_handle.png b/java/android/app/src/main/res/drawable-xhdpi/seekbar_handle.png new file mode 100644 index 0000000000..55f5f73991 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/seekbar_handle.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/seekbar_progress_dotted.png b/java/android/app/src/main/res/drawable-xhdpi/seekbar_progress_dotted.png new file mode 100644 index 0000000000..e6241d12e6 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/seekbar_progress_dotted.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/seekbar_thumb_invisible.png b/java/android/app/src/main/res/drawable-xhdpi/seekbar_thumb_invisible.png new file mode 100644 index 0000000000..acfe8d374a Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/seekbar_thumb_invisible.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/switch_side.png b/java/android/app/src/main/res/drawable-xhdpi/switch_side.png new file mode 100644 index 0000000000..3e6ae9a947 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/switch_side.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/switch_side_pressed.png b/java/android/app/src/main/res/drawable-xhdpi/switch_side_pressed.png new file mode 100644 index 0000000000..25e1522768 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/switch_side_pressed.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/take_picture.png b/java/android/app/src/main/res/drawable-xhdpi/take_picture.png new file mode 100644 index 0000000000..d6ced986e8 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/take_picture.png differ diff --git a/java/android/app/src/main/res/drawable-xhdpi/take_picture_pressed.png b/java/android/app/src/main/res/drawable-xhdpi/take_picture_pressed.png new file mode 100644 index 0000000000..5f9c8ee3b5 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xhdpi/take_picture_pressed.png differ diff --git a/java/android/app/src/main/res/drawable-xxhdpi-v4/btn_switch_default.png b/java/android/app/src/main/res/drawable-xxhdpi-v4/btn_switch_default.png new file mode 100644 index 0000000000..b9e66c7f60 Binary files /dev/null and b/java/android/app/src/main/res/drawable-xxhdpi-v4/btn_switch_default.png differ diff --git a/java/android/app/src/main/res/drawable-xxhdpi-v4/btn_switch_pressed.png b/java/android/app/src/main/res/drawable-xxhdpi-v4/btn_switch_pressed.png new file mode 100644 index 0000000000..9544133bda Binary files /dev/null and b/java/android/app/src/main/res/drawable-xxhdpi-v4/btn_switch_pressed.png differ diff --git a/java/android/app/src/main/res/drawable/btn_settings.xml b/java/android/app/src/main/res/drawable/btn_settings.xml new file mode 100644 index 0000000000..917897b999 --- /dev/null +++ b/java/android/app/src/main/res/drawable/btn_settings.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/java/android/app/src/main/res/drawable/btn_settings_default.xml b/java/android/app/src/main/res/drawable/btn_settings_default.xml new file mode 100644 index 0000000000..e19589a97e --- /dev/null +++ b/java/android/app/src/main/res/drawable/btn_settings_default.xml @@ -0,0 +1,13 @@ + + + + diff --git a/java/android/app/src/main/res/drawable/btn_settings_pressed.xml b/java/android/app/src/main/res/drawable/btn_settings_pressed.xml new file mode 100644 index 0000000000..c4af2a042d --- /dev/null +++ b/java/android/app/src/main/res/drawable/btn_settings_pressed.xml @@ -0,0 +1,13 @@ + + + + diff --git a/java/android/app/src/main/res/drawable/btn_shutter.xml b/java/android/app/src/main/res/drawable/btn_shutter.xml new file mode 100644 index 0000000000..4f9826d3ae --- /dev/null +++ b/java/android/app/src/main/res/drawable/btn_shutter.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/java/android/app/src/main/res/drawable/btn_shutter_default.xml b/java/android/app/src/main/res/drawable/btn_shutter_default.xml new file mode 100644 index 0000000000..234ca014a7 --- /dev/null +++ b/java/android/app/src/main/res/drawable/btn_shutter_default.xml @@ -0,0 +1,17 @@ + + + + + diff --git a/java/android/app/src/main/res/drawable/btn_shutter_pressed.xml b/java/android/app/src/main/res/drawable/btn_shutter_pressed.xml new file mode 100644 index 0000000000..accc7acedb --- /dev/null +++ b/java/android/app/src/main/res/drawable/btn_shutter_pressed.xml @@ -0,0 +1,17 @@ + + + + + diff --git a/java/android/app/src/main/res/drawable/btn_switch.xml b/java/android/app/src/main/res/drawable/btn_switch.xml new file mode 100644 index 0000000000..691e8c2e97 --- /dev/null +++ b/java/android/app/src/main/res/drawable/btn_switch.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/java/android/app/src/main/res/drawable/ic_launcher_background.xml b/java/android/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 0000000000..0d025f9bf6 --- /dev/null +++ b/java/android/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/android/app/src/main/res/layout-land/activity_main.xml b/java/android/app/src/main/res/layout-land/activity_main.xml new file mode 100644 index 0000000000..e3a175a9bc --- /dev/null +++ b/java/android/app/src/main/res/layout-land/activity_main.xml @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/activity_main.xml b/java/android/app/src/main/res/layout/activity_main.xml new file mode 100644 index 0000000000..72c4aa703f --- /dev/null +++ b/java/android/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,163 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/java/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 0000000000..eca70cfe52 --- /dev/null +++ b/java/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/java/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 0000000000..eca70cfe52 --- /dev/null +++ b/java/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/mipmap-hdpi/ic_launcher.png b/java/android/app/src/main/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 0000000000..898f3ed59a Binary files /dev/null and b/java/android/app/src/main/res/mipmap-hdpi/ic_launcher.png differ diff --git a/java/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png b/java/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png new file mode 100644 index 0000000000..dffca3601e Binary files /dev/null and b/java/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png differ diff --git a/java/android/app/src/main/res/mipmap-mdpi/ic_launcher.png b/java/android/app/src/main/res/mipmap-mdpi/ic_launcher.png new file mode 100644 index 0000000000..64ba76f75e Binary files /dev/null and b/java/android/app/src/main/res/mipmap-mdpi/ic_launcher.png differ diff --git a/java/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png b/java/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png new file mode 100644 index 0000000000..dae5e08234 Binary files /dev/null and b/java/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png differ diff --git a/java/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png b/java/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 0000000000..e5ed46597e Binary files /dev/null and b/java/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/java/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png b/java/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png new file mode 100644 index 0000000000..14ed0af350 Binary files /dev/null and b/java/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png differ diff --git a/java/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png b/java/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 0000000000..b0907cac3b Binary files /dev/null and b/java/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/java/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png b/java/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png new file mode 100644 index 0000000000..d8ae031549 Binary files /dev/null and b/java/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png differ diff --git a/java/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png b/java/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png new file mode 100644 index 0000000000..2c18de9e66 Binary files /dev/null and b/java/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/java/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png b/java/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png new file mode 100644 index 0000000000..beed3cdd2c Binary files /dev/null and b/java/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png differ diff --git a/java/android/app/src/main/res/values/arrays.xml b/java/android/app/src/main/res/values/arrays.xml new file mode 100644 index 0000000000..c7cf123788 --- /dev/null +++ b/java/android/app/src/main/res/values/arrays.xml @@ -0,0 +1,39 @@ + + + + 1 threads + 2 threads + 4 threads + 8 threads + + + 1 + 2 + 4 + 8 + + + HIGH(only big cores) + LOW(only LITTLE cores) + FULL(all cores) + NO_BIND(depends on system) + RAND_HIGH + RAND_LOW + + + LITE_POWER_HIGH + LITE_POWER_LOW + LITE_POWER_FULL + LITE_POWER_NO_BIND + LITE_POWER_RAND_HIGH + LITE_POWER_RAND_LOW + + + true + false + + + true + false + + \ No newline at end of file diff --git a/java/android/app/src/main/res/values/colors.xml b/java/android/app/src/main/res/values/colors.xml new file mode 100644 index 0000000000..f8ec1f0c3b --- /dev/null +++ b/java/android/app/src/main/res/values/colors.xml @@ -0,0 +1,22 @@ + + + #008577 + #00574B + #D81B60 + #FF000000 + #00000000 + #00000000 + #FFFFFFFF + + #000000 + #3B85F5 + #F5A623 + #FFFFFF + + #EEEEEE + + #3B85F5 + #333333 + #E5E5E5 + #3b85f5 + diff --git a/java/android/app/src/main/res/values/dimens.xml b/java/android/app/src/main/res/values/dimens.xml new file mode 100644 index 0000000000..2df89499da --- /dev/null +++ b/java/android/app/src/main/res/values/dimens.xml @@ -0,0 +1,17 @@ + + + 26dp + 36dp + 34dp + 60dp + 16dp + 67dp + 67dp + 56dp + 56dp + 46dp + 46dp + 32dp + 24dp + 16dp + diff --git a/java/android/app/src/main/res/values/strings.xml b/java/android/app/src/main/res/values/strings.xml new file mode 100644 index 0000000000..470605e836 --- /dev/null +++ b/java/android/app/src/main/res/values/strings.xml @@ -0,0 +1,28 @@ + + FastDeploy PicoDet + CHOOSE_INSTALLED_MODEL_KEY + MODEL_DIR_KEY + LABEL_PATH_KEY + CPU_THREAD_NUM_KEY + CPU_POWER_MODE_KEY + SCORE_THRESHOLD_KEY + ENABLE_LITE_FP16_MODE_KEY + models/picodet_s_320_coco_lcnet + labels/coco_label_list.txt + 2 + LITE_POWER_HIGH + 0.4 + true + + 拍照识别 + FD 实时识别 + < + 模型名称 + 识别结果 + 序号 + 名称 + 置信度 + 阈值控制 + 重新识别 + 保存结果 + \ No newline at end of file diff --git a/java/android/app/src/main/res/values/styles.xml b/java/android/app/src/main/res/values/styles.xml new file mode 100644 index 0000000000..67c1475944 --- /dev/null +++ b/java/android/app/src/main/res/values/styles.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/android/app/src/main/res/xml/settings.xml b/java/android/app/src/main/res/xml/settings.xml new file mode 100644 index 0000000000..26329068b6 --- /dev/null +++ b/java/android/app/src/main/res/xml/settings.xml @@ -0,0 +1,45 @@ + + + + + + + + + + diff --git a/java/android/build.gradle b/java/android/build.gradle new file mode 100644 index 0000000000..d8d678b3ff --- /dev/null +++ b/java/android/build.gradle @@ -0,0 +1,37 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +//plugins { +// id 'com.android.application' version '7.2.2' apply false +// id 'com.android.library' version '7.2.2' apply false +//} +// +//task clean(type: Delete) { +// delete rootProject.buildDir +//} + +buildscript { + repositories { + google() + jcenter() + // mavenCentral() + + } + dependencies { + classpath 'com.android.tools.build:gradle:7.2.2' + + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +allprojects { + repositories { + google() + jcenter() + // mavenCentral() + + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} diff --git a/java/android/fastdeploy/.gitignore b/java/android/fastdeploy/.gitignore new file mode 100644 index 0000000000..42afabfd2a --- /dev/null +++ b/java/android/fastdeploy/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/java/android/fastdeploy/build.gradle b/java/android/fastdeploy/build.gradle new file mode 100644 index 0000000000..cf4815a947 --- /dev/null +++ b/java/android/fastdeploy/build.gradle @@ -0,0 +1,84 @@ +import java.security.MessageDigest + +apply plugin: 'com.android.library' + + +android { + compileSdk 28 + + defaultConfig { + minSdk 15 + targetSdk 28 + + testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + consumerProguardFiles "consumer-rules.pro" + externalNativeBuild { + cmake { + arguments '-DANDROID_PLATFORM=android-21', '-DANDROID_STL=c++_shared', "-DANDROID_TOOLCHAIN=clang" + abiFilters 'armeabi-v7a', 'arm64-v8a' + cppFlags "-std=c++11" + } + } + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + externalNativeBuild { + cmake { + path "src/main/cpp/CMakeLists.txt" + version "3.10.2" + } + } + ndkVersion '20.1.5948944' +} + +dependencies { + + implementation 'com.android.support:appcompat-v7:28.0.0' + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'com.android.support.test:runner:1.0.2' + androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' +} + +def archives = [ + [ + 'src' : 'https://bj.bcebos.com/fastdeploy/test/fastdeploy-android-0.5.0-shared-dev.tgz', + 'dest': 'libs' + ] +] + +task downloadAndExtractLibs(type: DefaultTask) { + doFirst { + println "Downloading and extracting archives including libs ..." + } + doLast { + // Prepare cache folder for archives + String cachePath = "cache" + if (!file("${cachePath}").exists()) { + mkdir "${cachePath}" + } + archives.eachWithIndex { archive, index -> + MessageDigest messageDigest = MessageDigest.getInstance('MD5') + messageDigest.update(archive.src.bytes) + String cacheName = new BigInteger(1, messageDigest.digest()).toString(32) + // Download the target archive if not exists + boolean copyFiles = !file("${archive.dest}").exists() + if (!file("${cachePath}/${cacheName}.tgz").exists()) { + ant.get(src: archive.src, dest: file("${cachePath}/${cacheName}.tgz")) + copyFiles = true // force to copy files from the latest archive files + } + // Extract the target archive if its dest path does not exists + if (copyFiles) { + copy { + from tarTree("${cachePath}/${cacheName}.tgz") + into "${archive.dest}" + } + } + } + } +} +preBuild.dependsOn downloadAndExtractLibs \ No newline at end of file diff --git a/java/android/fastdeploy/consumer-rules.pro b/java/android/fastdeploy/consumer-rules.pro new file mode 100644 index 0000000000..e69de29bb2 diff --git a/java/android/fastdeploy/libs/.gitignore b/java/android/fastdeploy/libs/.gitignore new file mode 100644 index 0000000000..51958edc99 --- /dev/null +++ b/java/android/fastdeploy/libs/.gitignore @@ -0,0 +1 @@ +fastdeploy-* \ No newline at end of file diff --git a/java/android/fastdeploy/proguard-rules.pro b/java/android/fastdeploy/proguard-rules.pro new file mode 100644 index 0000000000..481bb43481 --- /dev/null +++ b/java/android/fastdeploy/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/java/android/fastdeploy/src/androidTest/java/com/baidu/paddle/fastdeploy/ExampleInstrumentedTest.java b/java/android/fastdeploy/src/androidTest/java/com/baidu/paddle/fastdeploy/ExampleInstrumentedTest.java new file mode 100644 index 0000000000..f453a9526a --- /dev/null +++ b/java/android/fastdeploy/src/androidTest/java/com/baidu/paddle/fastdeploy/ExampleInstrumentedTest.java @@ -0,0 +1,25 @@ +package com.baidu.paddle.fastdeploy; + +import android.content.Context; +import android.support.test.InstrumentationRegistry; +import android.support.test.runner.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); + assertEquals("com.baidu.paddle.fastdeploy.test", appContext.getPackageName()); + } +} \ No newline at end of file diff --git a/java/android/fastdeploy/src/main/AndroidManifest.xml b/java/android/fastdeploy/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..8fe6cbb858 --- /dev/null +++ b/java/android/fastdeploy/src/main/AndroidManifest.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/java/android/fastdeploy/src/main/cpp/CMakeLists.txt b/java/android/fastdeploy/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000000..09acd7cc8b --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/CMakeLists.txt @@ -0,0 +1,60 @@ +# For more information about using CMake with Android Studio, read the +# documentation: https://d.android.com/studio/projects/add-native-code.html + +# Sets the minimum version of CMake required to build the native library. +cmake_minimum_required(VERSION 3.10.2) + +# Declares and names the project. +project("fastdeploy_jni") + +# Creates and names a library, sets it as either STATIC +# or SHARED, and provides the relative paths to its source code. +# You can define multiple libraries, and CMake builds them for you. +# Gradle automatically packages shared libraries with your APK. + +set(FastDeploy_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../libs/fastdeploy-android-0.5.0-shared-dev") + +find_package(FastDeploy REQUIRED) + +include_directories(.) +include_directories(${FastDeploy_INCLUDE_DIRS}) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math -Ofast -Os -DNDEBUG -fomit-frame-pointer -fno-asynchronous-unwind-tables -fno-unwind-tables") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -fvisibility-inlines-hidden -fdata-sections -ffunction-sections") +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,-z,nocopyreloc") + +add_library( + fastdeploy_jni + SHARED + utils_jni.cc + bitmap_jni.cc + vision/results_jni.cc + vision/visualize_jni.cc + vision/detection/picodet_jni.cc + vision/classification/paddleclas_model_jni.cc) + +# Searches for a specified prebuilt library and stores the path as a +# variable. Because CMake includes system libraries in the search path by +# default, you only need to specify the name of the public NDK library +# you want to add. CMake verifies that the library exists before +# completing its build. + +find_library( # Sets the name of the path variable. + log-lib + # Specifies the name of the NDK library that + # you want CMake to locate. + log) + +# Specifies libraries CMake should link to your target library. You can link +# multiple libraries, such as libraries you define in this build script, +# prebuilt third-party libraries, or system libraries. + +target_link_libraries( + # Specifies the target library. + fastdeploy_jni + jnigraphics + ${FASTDEPLOY_LIBS} + GLESv2 + EGL + ${log-lib} +) diff --git a/java/android/fastdeploy/src/main/cpp/bitmap_jni.cc b/java/android/fastdeploy/src/main/cpp/bitmap_jni.cc new file mode 100644 index 0000000000..5a3060911a --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/bitmap_jni.cc @@ -0,0 +1,100 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "bitmap_jni.h" // NOLINT + +#include // NOLINT + +#include "utils_jni.h" // NOLINT + +namespace fastdeploy { +namespace jni { + +jboolean ARGB888Bitmap2RGBA(JNIEnv *env, jobject j_argb8888_bitmap, + cv::Mat *c_rgba) { + // Convert the android bitmap(ARGB8888) to the OpenCV RGBA image. Actually, + // the data layout of ARGB8888 is R, G, B, A, it's the same as CV RGBA image, + // so it is unnecessary to do the conversion of color format, check + // https://developer.android.com/reference/android/graphics/Bitmap.Config#ARGB_8888 + // to get the more details about Bitmap.Config.ARGB8888 + AndroidBitmapInfo j_bitmap_info; + if (AndroidBitmap_getInfo(env, j_argb8888_bitmap, &j_bitmap_info) < 0) { + LOGE("Invoke AndroidBitmap_getInfo() failed!"); + return JNI_FALSE; + } + if (j_bitmap_info.format != ANDROID_BITMAP_FORMAT_RGBA_8888) { + LOGE("Only Bitmap.Config.ARGB8888 color format is supported!"); + return JNI_FALSE; + } + void *j_bitmap_pixels; + if (AndroidBitmap_lockPixels(env, j_argb8888_bitmap, &j_bitmap_pixels) < 0) { + LOGE("Invoke AndroidBitmap_lockPixels() failed!"); + return JNI_FALSE; + } + cv::Mat j_bitmap_im(static_cast(j_bitmap_info.height), + static_cast(j_bitmap_info.width), CV_8UC4, + j_bitmap_pixels); + j_bitmap_im.copyTo(*(c_rgba)); + if (AndroidBitmap_unlockPixels(env, j_argb8888_bitmap) < 0) { + LOGE("Invoke AndroidBitmap_unlockPixels() failed!"); + return JNI_FALSE; + } + return JNI_TRUE; +} + +jboolean ARGB888Bitmap2BGR(JNIEnv *env, jobject j_argb8888_bitmap, + cv::Mat *c_bgr) { + cv::Mat c_rgba; + if (!ARGB888Bitmap2RGBA(env, j_argb8888_bitmap, &c_rgba)) { + return JNI_FALSE; + } + cv::cvtColor(c_rgba, *(c_bgr), cv::COLOR_RGBA2BGR); + return JNI_TRUE; +} + +jboolean RGBA2ARGB888Bitmap(JNIEnv *env, jobject j_argb8888_bitmap, + const cv::Mat &c_rgba) { + AndroidBitmapInfo j_bitmap_info; + if (AndroidBitmap_getInfo(env, j_argb8888_bitmap, &j_bitmap_info) < 0) { + LOGE("Invoke AndroidBitmap_getInfo() failed!"); + return JNI_FALSE; + } + void *j_bitmap_pixels; + if (AndroidBitmap_lockPixels(env, j_argb8888_bitmap, &j_bitmap_pixels) < 0) { + LOGE("Invoke AndroidBitmap_lockPixels() failed!"); + return JNI_FALSE; + } + cv::Mat j_bitmap_im(static_cast(j_bitmap_info.height), + static_cast(j_bitmap_info.width), CV_8UC4, + j_bitmap_pixels); + c_rgba.copyTo(j_bitmap_im); + if (AndroidBitmap_unlockPixels(env, j_argb8888_bitmap) < 0) { + LOGE("Invoke AndroidBitmap_unlockPixels() failed!"); + return JNI_FALSE; + } + return JNI_TRUE; +} + +jboolean BGR2ARGB888Bitmap(JNIEnv *env, jobject j_argb8888_bitmap, + const cv::Mat &c_bgr) { + if (c_bgr.empty()) { + return JNI_FALSE; + } + cv::Mat c_rgba; + cv::cvtColor(c_bgr, c_rgba, cv::COLOR_BGR2RGBA); + return RGBA2ARGB888Bitmap(env, j_argb8888_bitmap, c_rgba); +} + +} // namespace jni +} // namespace fastdeploy diff --git a/java/android/fastdeploy/src/main/cpp/bitmap_jni.h b/java/android/fastdeploy/src/main/cpp/bitmap_jni.h new file mode 100644 index 0000000000..5bd8b195a3 --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/bitmap_jni.h @@ -0,0 +1,39 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include // NOLINT + +#include "fastdeploy/vision.h" // NOLINT + +namespace fastdeploy { +namespace jni { + +// Convert the android bitmap(ARGB8888) to the OpenCV RGBA image. Actually, +// the data layout of ARGB8888 is R, G, B, A, it's the same as CV RGBA image, +// so it is unnecessary to do the conversion of color format, check +// https://developer.android.com/reference/android/graphics/Bitmap.Config#ARGB_8888 +// to get the more details about Bitmap.Config.ARGB8888 +jboolean ARGB888Bitmap2RGBA(JNIEnv *env, jobject j_argb8888_bitmap, + cv::Mat *c_rgba); +jboolean RGBA2ARGB888Bitmap(JNIEnv *env, jobject j_argb8888_bitmap, + const cv::Mat &c_rgba); +jboolean ARGB888Bitmap2BGR(JNIEnv *env, jobject j_argb8888_bitmap, + cv::Mat *c_bgr); +jboolean BGR2ARGB888Bitmap(JNIEnv *env, jobject j_argb8888_bitmap, + const cv::Mat &c_bgr); + +} // namespace jni +} // namespace fastdeploy diff --git a/java/android/fastdeploy/src/main/cpp/convert_jni.h b/java/android/fastdeploy/src/main/cpp/convert_jni.h new file mode 100644 index 0000000000..31ee92fe75 --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/convert_jni.h @@ -0,0 +1,140 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include // NOLINT + +#include // NOLINT +#include // NOLINT + +namespace fastdeploy { +namespace jni { + +template +OutputType ConvertTo(JNIEnv *env, InputType input); + +template +OutputType ConvertTo(JNIEnv *env, const InputType *input, int64_t len); + +/// jstring -> std::string +template <> +inline std::string ConvertTo(JNIEnv *env, jstring jstr) { + // In java, a unicode char will be encoded using 2 bytes (utf16). + // so jstring will contain characters utf16. std::string in c++ is + // essentially a string of bytes, not characters, so if we want to + // pass jstring from JNI to c++, we have convert utf16 to bytes. + if (!jstr) { + return ""; + } + const jclass jstring_clazz = env->GetObjectClass(jstr); + const jmethodID getBytesID = + env->GetMethodID(jstring_clazz, "getBytes", "(Ljava/lang/String;)[B"); + const jbyteArray jstring_bytes = (jbyteArray)env->CallObjectMethod( + jstr, getBytesID, env->NewStringUTF("UTF-8")); + + size_t length = static_cast(env->GetArrayLength(jstring_bytes)); + jbyte *jstring_bytes_ptr = env->GetByteArrayElements(jstring_bytes, NULL); + + std::string res = + std::string(reinterpret_cast(jstring_bytes_ptr), length); + env->ReleaseByteArrayElements(jstring_bytes, jstring_bytes_ptr, JNI_ABORT); + + env->DeleteLocalRef(jstring_bytes); + env->DeleteLocalRef(jstring_clazz); + return res; +} + +/// std::string -> jstring +template <> +inline jstring ConvertTo(JNIEnv *env, std::string str) { + auto *cstr_data_ptr = str.c_str(); + jclass jstring_clazz = env->FindClass("java/lang/String"); + jmethodID initID = + env->GetMethodID(jstring_clazz, "", "([BLjava/lang/String;)V"); + + jbyteArray jstring_bytes = env->NewByteArray(strlen(cstr_data_ptr)); + env->SetByteArrayRegion(jstring_bytes, 0, strlen(cstr_data_ptr), + reinterpret_cast(cstr_data_ptr)); + + jstring jstring_encoding = env->NewStringUTF("UTF-8"); + jstring res = (jstring)(env->NewObject(jstring_clazz, initID, jstring_bytes, + jstring_encoding)); + + env->DeleteLocalRef(jstring_clazz); + env->DeleteLocalRef(jstring_bytes); + env->DeleteLocalRef(jstring_encoding); + + return res; +} + +/// jlongArray -> std::vector +template <> +inline std::vector ConvertTo(JNIEnv *env, jlongArray jdata) { + int jdata_size = env->GetArrayLength(jdata); + jlong *jdata_ptr = env->GetLongArrayElements(jdata, nullptr); + std::vector res(jdata_ptr, jdata_ptr + jdata_size); + env->ReleaseLongArrayElements(jdata, jdata_ptr, 0); + return res; +} + +/// jfloatArray -> std::vector +template <> +inline std::vector ConvertTo(JNIEnv *env, jfloatArray jdata) { + int jdata_size = env->GetArrayLength(jdata); + jfloat *jdata_ptr = env->GetFloatArrayElements(jdata, nullptr); + std::vector res(jdata_ptr, jdata_ptr + jdata_size); + env->ReleaseFloatArrayElements(jdata, jdata_ptr, 0); + return res; +} + +/// std::vector -> jlongArray +template <> +inline jlongArray ConvertTo(JNIEnv *env, const std::vector &cvec) { + jlongArray res = env->NewLongArray(cvec.size()); + jlong *jbuf = new jlong[cvec.size()]; + for (size_t i = 0; i < cvec.size(); ++i) { + jbuf[i] = (jlong)cvec[i]; + } + env->SetLongArrayRegion(res, 0, cvec.size(), jbuf); + delete[] jbuf; + return res; +} + +/// cxx float buffer -> jfloatArray +template <> +inline jfloatArray ConvertTo(JNIEnv *env, const float *cbuf, int64_t len) { + jfloatArray res = env->NewFloatArray(len); + env->SetFloatArrayRegion(res, 0, len, cbuf); + return res; +} + +/// cxx int buffer -> jintArray +template <> +inline jintArray ConvertTo(JNIEnv *env, const int *cbuf, int64_t len) { + jintArray res = env->NewIntArray(len); + env->SetIntArrayRegion(res, 0, len, cbuf); + return res; +} + +/// cxx int8_t buffer -> jbyteArray +template <> +inline jbyteArray ConvertTo(JNIEnv *env, const int8_t *cbuf, int64_t len) { + jbyteArray res = env->NewByteArray(len); + env->SetByteArrayRegion(res, 0, len, cbuf); + return res; +} + +} // namespace jni +} // namespace fastdeploy diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni.h b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni.h new file mode 100644 index 0000000000..2b40a87d2b --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni.h @@ -0,0 +1,18 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "bitmap_jni.h" // NOLINT +#include "convert_jni.h" // NOLINT +#include "utils_jni.h" // NOLINT diff --git a/java/android/fastdeploy/src/main/cpp/utils_jni.cc b/java/android/fastdeploy/src/main/cpp/utils_jni.cc new file mode 100644 index 0000000000..47775f03cf --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/utils_jni.cc @@ -0,0 +1,82 @@ +// +// Created by qiuyanjun on 2022/10/19. +// + +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utils_jni.h" + +namespace fastdeploy { +namespace jni { + +// Assets Loader Utils. +bool AssetsLoaderUtils::detection_labels_loaded_ = false; +bool AssetsLoaderUtils::classification_labels_loaded_ = false; +std::vector AssetsLoaderUtils::detection_labels_ = {}; +std::vector AssetsLoaderUtils::classification_labels_ = {}; + +bool AssetsLoaderUtils::IsDetectionLabelsLoaded() { + return detection_labels_loaded_; +} + +bool AssetsLoaderUtils::IsClassificationLabelsLoaded() { + return classification_labels_loaded_; +} + +const std::vector& AssetsLoaderUtils::GetDetectionLabels() { + return detection_labels_; +} + +const std::vector& AssetsLoaderUtils::GetClassificationLabels() { + return classification_labels_; +} + +void AssetsLoaderUtils::LoadClassificationLabels(const std::string& path, + bool force_reload) { + if (force_reload || (!classification_labels_loaded_)) { + classification_labels_loaded_ = + LoadLabelsFromTxt(path, &classification_labels_); + } +} + +void AssetsLoaderUtils::LoadDetectionLabels(const std::string& path, + bool force_reload) { + if (force_reload || (!detection_labels_loaded_)) { + detection_labels_loaded_ = LoadLabelsFromTxt(path, &detection_labels_); + } +} + +bool AssetsLoaderUtils::LoadLabelsFromTxt(const std::string& txt_path, + std::vector* labels) { + labels->clear(); + std::ifstream file; + file.open(txt_path); + if (!file.is_open()) { + return false; + } + while (file) { + std::string line; + std::getline(file, line); + if (!line.empty() && line != "\n") { + labels->push_back(line); + } + } + file.clear(); + file.close(); + return labels->size() > 0; +} + +} // namespace jni +} // namespace fastdeploy \ No newline at end of file diff --git a/java/android/fastdeploy/src/main/cpp/utils_jni.h b/java/android/fastdeploy/src/main/cpp/utils_jni.h new file mode 100644 index 0000000000..8aeabe10ad --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/utils_jni.h @@ -0,0 +1,80 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#ifdef __ANDROID__ +#include // NOLINT +#endif +#include // NOLINT +#include // NOLINT +#include // NOLINT + +#define TAG "[FastDeploy][JNI]" +#ifdef __ANDROID__ +#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__) +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) +#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) +#define LOGF(...) __android_log_print(ANDROID_LOG_FATAL, TAG, __VA_ARGS__) +#else +#define LOGD(...) \ + {} +#define LOGI(...) \ + {} +#define LOGW(...) \ + {} +#define LOGE(...) \ + {} +#define LOGF(...) \ + {} +#endif + +namespace fastdeploy { +namespace jni { + +inline int64_t GetCurrentTime() { + struct timeval time; + gettimeofday(&time, NULL); + return 1000000LL * (int64_t)time.tv_sec + (int64_t)time.tv_usec; +} + +inline double GetElapsedTime(int64_t time) { + return (GetCurrentTime() - time) / 1000.0f; +} + +class AssetsLoaderUtils { + public: + static bool detection_labels_loaded_; + static bool classification_labels_loaded_; + static std::vector detection_labels_; + static std::vector classification_labels_; + + public: + static bool IsDetectionLabelsLoaded(); + static bool IsClassificationLabelsLoaded(); + static const std::vector& GetDetectionLabels(); + static const std::vector& GetClassificationLabels(); + static void LoadClassificationLabels(const std::string& path, + bool force_reload = false); + static void LoadDetectionLabels(const std::string& path, + bool force_reload = false); + + private: + static bool LoadLabelsFromTxt(const std::string& txt_path, + std::vector* labels); +}; + +} // namespace jni +} // namespace fastdeploy diff --git a/java/android/fastdeploy/src/main/cpp/vision/classification/paddleclas_model_jni.cc b/java/android/fastdeploy/src/main/cpp/vision/classification/paddleclas_model_jni.cc new file mode 100644 index 0000000000..11600d9a0b --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/vision/classification/paddleclas_model_jni.cc @@ -0,0 +1,148 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include // NOLINT + +#include "fastdeploy_jni.h" // NOLINT + +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jlong JNICALL +Java_com_baidu_paddle_fastdeploy_vision_classification_PaddleClasModel_bindNative( + JNIEnv *env, jclass clazz, jstring model_file, jstring params_file, + jstring config_file, jint cpu_num_thread, jboolean enable_lite_fp16, + jint lite_power_mode, jstring lite_optimized_model_dir, + jboolean enable_record_time_of_runtime, jstring label_file) { + std::string c_model_file = + fastdeploy::jni::ConvertTo(env, model_file); + std::string c_params_file = + fastdeploy::jni::ConvertTo(env, params_file); + std::string c_config_file = + fastdeploy::jni::ConvertTo(env, config_file); + std::string c_label_file = + fastdeploy::jni::ConvertTo(env, label_file); + std::string c_lite_optimized_model_dir = + fastdeploy::jni::ConvertTo(env, lite_optimized_model_dir); + auto c_cpu_num_thread = static_cast(cpu_num_thread); + auto c_enable_lite_fp16 = static_cast(enable_lite_fp16); + auto c_lite_power_mode = + static_cast(lite_power_mode); + fastdeploy::RuntimeOption c_option; + c_option.UseCpu(); + c_option.UseLiteBackend(); + c_option.SetCpuThreadNum(c_cpu_num_thread); + c_option.SetLitePowerMode(c_lite_power_mode); + c_option.SetLiteOptimizedModelDir(c_lite_optimized_model_dir); + if (c_enable_lite_fp16) { + c_option.EnableLiteFP16(); + } + auto c_model_ptr = new fastdeploy::vision::classification::PaddleClasModel( + c_model_file, c_params_file, c_config_file, c_option); + // Enable record Runtime time costs. + if (enable_record_time_of_runtime) { + c_model_ptr->EnableRecordTimeOfRuntime(); + } + // Load classification labels if label path is not empty. + if ((!fastdeploy::jni::AssetsLoaderUtils::IsClassificationLabelsLoaded()) && + (!c_label_file.empty())) { + fastdeploy::jni::AssetsLoaderUtils::LoadClassificationLabels(c_label_file); + } + // WARN: need to release manually in Java ! + return reinterpret_cast(c_model_ptr); // native model context +} + +JNIEXPORT jlong JNICALL +Java_com_baidu_paddle_fastdeploy_vision_classification_PaddleClasModel_predictNative( + JNIEnv *env, jclass clazz, jlong native_model_context, + jobject argb8888_bitmap, jboolean saved, jstring saved_image_path, + jfloat score_threshold, jboolean rendering) { + if (native_model_context == 0) { + return 0; + } + cv::Mat c_bgr; + auto t = fastdeploy::jni::GetCurrentTime(); + if (!fastdeploy::jni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { + return 0; + } + LOGD("Read from bitmap costs %f ms", fastdeploy::jni::GetElapsedTime(t)); + auto c_model_ptr = + reinterpret_cast( + native_model_context); + auto c_result_ptr = new fastdeploy::vision::ClassifyResult(); + t = fastdeploy::jni::GetCurrentTime(); + if (!c_model_ptr->Predict(&c_bgr, c_result_ptr, 100)) { + delete c_result_ptr; + return 0; + } + LOGD("Predict from native costs %f ms", fastdeploy::jni::GetElapsedTime(t)); + if (c_model_ptr->EnabledRecordTimeOfRuntime()) { + auto info_of_runtime = c_model_ptr->PrintStatisInfoOfRuntime(); + LOGD("Avg runtime costs %f ms", info_of_runtime["avg_time"] * 1000.0f); + } + if (!c_result_ptr->scores.empty() && rendering) { + t = fastdeploy::jni::GetCurrentTime(); + cv::Mat c_vis_im; + if (fastdeploy::jni::AssetsLoaderUtils::IsClassificationLabelsLoaded()) { + c_vis_im = fastdeploy::vision::VisClassification( + c_bgr, *(c_result_ptr), + fastdeploy::jni::AssetsLoaderUtils::GetClassificationLabels(), 5, + score_threshold, 1.0f); + } else { + c_vis_im = fastdeploy::vision::VisClassification( + c_bgr, *(c_result_ptr), 5, score_threshold, 1.0f); + } + LOGD("Visualize from native costs %f ms", + fastdeploy::jni::GetElapsedTime(t)); + // Rendering to bitmap + t = fastdeploy::jni::GetCurrentTime(); + if (!fastdeploy::jni::BGR2ARGB888Bitmap(env, argb8888_bitmap, c_vis_im)) { + delete c_result_ptr; + return 0; + } + LOGD("Write to bitmap from native costs %f ms", + fastdeploy::jni::GetElapsedTime(t)); + std::string c_saved_image_path = + fastdeploy::jni::ConvertTo(env, saved_image_path); + if (!c_saved_image_path.empty() && saved) { + t = fastdeploy::jni::GetCurrentTime(); + cv::imwrite(c_saved_image_path, c_bgr); + LOGD("Save image from native costs %f ms, path: %s", + fastdeploy::jni::GetElapsedTime(t), c_saved_image_path.c_str()); + } + } + // WARN: need to release it manually in Java ! + return reinterpret_cast(c_result_ptr); // native result context +} + +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_classification_PaddleClasModel_releaseNative( + JNIEnv *env, jclass clazz, jlong native_model_context) { + auto c_model_ptr = + reinterpret_cast( + native_model_context); + if (c_model_ptr->EnabledRecordTimeOfRuntime()) { + auto info_of_runtime = c_model_ptr->PrintStatisInfoOfRuntime(); + LOGD("[End] Avg runtime costs %f ms", + info_of_runtime["avg_time"] * 1000.0f); + } + delete c_model_ptr; + LOGD("[End] Release PaddleClasModel in native !"); + return JNI_TRUE; +} + +#ifdef __cplusplus +} +#endif diff --git a/java/android/fastdeploy/src/main/cpp/vision/detection/picodet_jni.cc b/java/android/fastdeploy/src/main/cpp/vision/detection/picodet_jni.cc new file mode 100644 index 0000000000..1e4d790e3f --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/vision/detection/picodet_jni.cc @@ -0,0 +1,149 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include // NOLINT + +#include "fastdeploy_jni.h" // NOLINT + +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jlong JNICALL +Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_bindNative( + JNIEnv *env, jclass clazz, jstring model_file, jstring params_file, + jstring config_file, jint cpu_num_thread, jboolean enable_lite_fp16, + jint lite_power_mode, jstring lite_optimized_model_dir, + jboolean enable_record_time_of_runtime, jstring label_file) { + std::string c_model_file = + fastdeploy::jni::ConvertTo(env, model_file); + std::string c_params_file = + fastdeploy::jni::ConvertTo(env, params_file); + std::string c_config_file = + fastdeploy::jni::ConvertTo(env, config_file); + std::string c_label_file = + fastdeploy::jni::ConvertTo(env, label_file); + std::string c_lite_optimized_model_dir = + fastdeploy::jni::ConvertTo(env, lite_optimized_model_dir); + auto c_cpu_num_thread = static_cast(cpu_num_thread); + auto c_enable_lite_fp16 = static_cast(enable_lite_fp16); + auto c_lite_power_mode = + static_cast(lite_power_mode); + fastdeploy::RuntimeOption c_option; + c_option.UseCpu(); + c_option.UseLiteBackend(); + c_option.SetCpuThreadNum(c_cpu_num_thread); + c_option.SetLitePowerMode(c_lite_power_mode); + c_option.SetLiteOptimizedModelDir(c_lite_optimized_model_dir); + if (c_enable_lite_fp16) { + c_option.EnableLiteFP16(); + } + auto c_model_ptr = new fastdeploy::vision::detection::PicoDet( + c_model_file, c_params_file, c_config_file, c_option); + // Enable record Runtime time costs. + if (enable_record_time_of_runtime) { + c_model_ptr->EnableRecordTimeOfRuntime(); + } + // Load detection labels if label path is not empty. + if ((!fastdeploy::jni::AssetsLoaderUtils::IsDetectionLabelsLoaded()) && + (!c_label_file.empty())) { + fastdeploy::jni::AssetsLoaderUtils::LoadDetectionLabels(c_label_file); + } + // WARN: need to release manually in Java ! + return reinterpret_cast(c_model_ptr); // native model context +} + +JNIEXPORT jlong JNICALL +Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_predictNative( + JNIEnv *env, jclass clazz, jlong native_model_context, + jobject argb8888_bitmap, jboolean saved, jstring saved_image_path, + jfloat score_threshold, jboolean rendering) { + if (native_model_context == 0) { + return 0; + } + cv::Mat c_bgr; + auto t = fastdeploy::jni::GetCurrentTime(); + if (!fastdeploy::jni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { + return 0; + } + LOGD("Read from bitmap costs %f ms", fastdeploy::jni::GetElapsedTime(t)); + auto c_model_ptr = reinterpret_cast( + native_model_context); + auto c_result_ptr = new fastdeploy::vision::DetectionResult(); + t = fastdeploy::jni::GetCurrentTime(); + if (!c_model_ptr->Predict(&c_bgr, c_result_ptr)) { + delete c_result_ptr; + return 0; + } + LOGD("Predict from native costs %f ms", fastdeploy::jni::GetElapsedTime(t)); + if (c_model_ptr->EnabledRecordTimeOfRuntime()) { + auto info_of_runtime = c_model_ptr->PrintStatisInfoOfRuntime(); + LOGD("Avg runtime costs %f ms", info_of_runtime["avg_time"] * 1000.0f); + } + if (!c_result_ptr->boxes.empty() && rendering) { + t = fastdeploy::jni::GetCurrentTime(); + cv::Mat c_vis_im; + if (fastdeploy::jni::AssetsLoaderUtils::IsDetectionLabelsLoaded()) { + c_vis_im = fastdeploy::vision::VisDetection( + c_bgr, *(c_result_ptr), + fastdeploy::jni::AssetsLoaderUtils::GetDetectionLabels(), + score_threshold, 2, 1.0f); + } else { + c_vis_im = fastdeploy::vision::VisDetection(c_bgr, *(c_result_ptr), + score_threshold, 2, 1.0f); + } + LOGD("Visualize from native costs %f ms", + fastdeploy::jni::GetElapsedTime(t)); + // Rendering to bitmap + t = fastdeploy::jni::GetCurrentTime(); + if (!fastdeploy::jni::BGR2ARGB888Bitmap(env, argb8888_bitmap, c_vis_im)) { + delete c_result_ptr; + return 0; + } + LOGD("Write to bitmap from native costs %f ms", + fastdeploy::jni::GetElapsedTime(t)); + std::string c_saved_image_path = + fastdeploy::jni::ConvertTo(env, saved_image_path); + if (!c_saved_image_path.empty() && saved) { + t = fastdeploy::jni::GetCurrentTime(); + cv::imwrite(c_saved_image_path, c_vis_im); + LOGD("Save image from native costs %f ms, path: %s", + fastdeploy::jni::GetElapsedTime(t), c_saved_image_path.c_str()); + } + } + // WARN: need to release it manually in Java ! + return reinterpret_cast(c_result_ptr); // native result context +} + +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_releaseNative( + JNIEnv *env, jclass clazz, jlong native_model_context) { + if (native_model_context == 0) { + return JNI_FALSE; + } + auto c_model_ptr = reinterpret_cast( + native_model_context); + if (c_model_ptr->EnabledRecordTimeOfRuntime()) { + auto info_of_runtime = c_model_ptr->PrintStatisInfoOfRuntime(); + LOGD("[End] Avg runtime costs %f ms", + info_of_runtime["avg_time"] * 1000.0f); + } + delete c_model_ptr; + LOGD("[End] Release PicoDet in native !"); + return JNI_TRUE; +} + +#ifdef __cplusplus +} +#endif diff --git a/java/android/fastdeploy/src/main/cpp/vision/results_jni.cc b/java/android/fastdeploy/src/main/cpp/vision/results_jni.cc new file mode 100644 index 0000000000..8fd224de9b --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/vision/results_jni.cc @@ -0,0 +1,132 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include // NOLINT +#include // NOLINT + +#include "fastdeploy/vision.h" // NOLINT +#include "fastdeploy_jni.h" // NOLINT + +#ifdef __cplusplus +extern "C" { +#endif + +/// Native DetectionResult for vision::DetectionResult. +JNIEXPORT jint JNICALL +Java_com_baidu_paddle_fastdeploy_vision_DetectionResult_copyBoxesNumFromNative( + JNIEnv *env, jclass clazz, jlong native_result_context) { + auto c_result_ptr = reinterpret_cast( + native_result_context); + return static_cast(c_result_ptr->boxes.size()); +} + +JNIEXPORT jfloatArray JNICALL +Java_com_baidu_paddle_fastdeploy_vision_DetectionResult_copyBoxesFromNative( + JNIEnv *env, jclass clazz, jlong native_result_context) { + auto c_result_ptr = reinterpret_cast( + native_result_context); + if (c_result_ptr->boxes.empty()) { + return {}; + } + const auto len = static_cast(c_result_ptr->boxes.size()); + float buffer[len * 4]; + const auto &boxes = c_result_ptr->boxes; + for (int64_t i = 0; i < len; ++i) { + std::memcpy((buffer + i * 4), (boxes.at(i).data()), 4 * sizeof(float)); + } + return fastdeploy::jni::ConvertTo(env, buffer, len * 4); +} + +JNIEXPORT jfloatArray JNICALL +Java_com_baidu_paddle_fastdeploy_vision_DetectionResult_copyScoresFromNative( + JNIEnv *env, jclass clazz, jlong native_result_context) { + auto c_result_ptr = reinterpret_cast( + native_result_context); + if (c_result_ptr->scores.empty()) { + return {}; + } + const auto len = static_cast(c_result_ptr->scores.size()); + const float *buffer = static_cast(c_result_ptr->scores.data()); + return fastdeploy::jni::ConvertTo(env, buffer, len); +} + +JNIEXPORT jintArray JNICALL +Java_com_baidu_paddle_fastdeploy_vision_DetectionResult_copyLabelIdsFromNative( + JNIEnv *env, jclass clazz, jlong native_result_context) { + auto c_result_ptr = reinterpret_cast( + native_result_context); + if (c_result_ptr->label_ids.empty()) { + return {}; + } + const auto len = static_cast(c_result_ptr->label_ids.size()); + const int *buffer = static_cast(c_result_ptr->label_ids.data()); + return fastdeploy::jni::ConvertTo(env, buffer, len); +} + +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_DetectionResult_releaseNative( + JNIEnv *env, jclass clazz, jlong native_result_context) { + if (native_result_context == 0) { + return JNI_FALSE; + } + auto c_result_ptr = reinterpret_cast( + native_result_context); + delete c_result_ptr; + LOGD("Release DetectionResult in native !"); + return JNI_TRUE; +} + +/// Native ClassifyResult for vision::ClassifyResult. +JNIEXPORT jfloatArray JNICALL +Java_com_baidu_paddle_fastdeploy_vision_ClassifyResult_copyScoresFromNative( + JNIEnv *env, jclass clazz, jlong native_result_context) { + auto c_result_ptr = reinterpret_cast( + native_result_context); + if (c_result_ptr->scores.empty()) { + return {}; + } + const auto len = static_cast(c_result_ptr->scores.size()); + const float *buffer = static_cast(c_result_ptr->scores.data()); + return fastdeploy::jni::ConvertTo(env, buffer, len); +} + +JNIEXPORT jintArray JNICALL +Java_com_baidu_paddle_fastdeploy_vision_ClassifyResult_copyLabelIdsFromNative( + JNIEnv *env, jclass clazz, jlong native_result_context) { + auto c_result_ptr = reinterpret_cast( + native_result_context); + if (c_result_ptr->label_ids.empty()) { + return {}; + } + const auto len = static_cast(c_result_ptr->label_ids.size()); + const int *buffer = static_cast(c_result_ptr->label_ids.data()); + return fastdeploy::jni::ConvertTo(env, buffer, len); +} + +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_ClassifyResult_releaseNative( + JNIEnv *env, jclass clazz, jlong native_result_context) { + if (native_result_context == 0) { + return JNI_FALSE; + } + auto c_result_ptr = reinterpret_cast( + native_result_context); + delete c_result_ptr; + LOGD("Release ClassifyResult in native !"); + return JNI_TRUE; +} + +#ifdef __cplusplus +} +#endif diff --git a/java/android/fastdeploy/src/main/cpp/vision/visualize_jni.cc b/java/android/fastdeploy/src/main/cpp/vision/visualize_jni.cc new file mode 100644 index 0000000000..8f54073007 --- /dev/null +++ b/java/android/fastdeploy/src/main/cpp/vision/visualize_jni.cc @@ -0,0 +1,92 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include + +#include "fastdeploy_jni.h" + +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_Visualize_visDetectionNative( + JNIEnv *env, jclass clazz, jobject argb8888_bitmap, jobjectArray boxes, + jfloatArray scores, jintArray label_ids, jfloat score_threshold, + jint line_size, jfloat font_size, jobjectArray labels) { + // Draw DetectionResult to ARGB8888 Bitmap + int len = env->GetArrayLength(boxes); + if ((len == 0) || (len != env->GetArrayLength(scores)) || + (len != env->GetArrayLength(label_ids))) { + return JNI_FALSE; + } + fastdeploy::vision::DetectionResult c_result; + c_result.Resize(len); + bool check_validation = true; + for (int i = 0; i < len; ++i) { + auto j_box = + reinterpret_cast(env->GetObjectArrayElement(boxes, i)); + if (env->GetArrayLength(j_box) == 4) { + jfloat *j_box_ptr = env->GetFloatArrayElements(j_box, nullptr); + std::memcpy(c_result.boxes[i].data(), j_box_ptr, 4 * sizeof(float)); + env->ReleaseFloatArrayElements(j_box, j_box_ptr, 0); + } else { + check_validation = false; + break; + } + } + if (!check_validation) { + return JNI_FALSE; + } + jfloat *j_scores_ptr = env->GetFloatArrayElements(scores, nullptr); + std::memcpy(c_result.scores.data(), j_scores_ptr, len * sizeof(float)); + env->ReleaseFloatArrayElements(scores, j_scores_ptr, 0); + jint *j_label_ids_ptr = env->GetIntArrayElements(label_ids, nullptr); + std::memcpy(c_result.label_ids.data(), j_label_ids_ptr, len * sizeof(int)); + env->ReleaseIntArrayElements(label_ids, j_label_ids_ptr, 0); + + // Get labels from Java + std::vector c_labels; + int label_len = env->GetArrayLength(labels); + if (label_len > 0) { + c_labels.reserve(label_len); + for (int i = 0; i < label_len; ++i) { + auto j_str = + reinterpret_cast(env->GetObjectArrayElement(labels, i)); + c_labels.push_back(fastdeploy::jni::ConvertTo(env, j_str)); + } + } + + cv::Mat c_bgr; + // From ARGB Bitmap to BGR + if (!fastdeploy::jni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { + return JNI_FALSE; + } + cv::Mat c_vis_im; + if (!c_labels.empty()) { + c_vis_im = fastdeploy::vision::VisDetection( + c_bgr, c_result, c_labels, score_threshold, line_size, font_size); + } else { + c_vis_im = fastdeploy::vision::VisDetection( + c_bgr, c_result, score_threshold, line_size, font_size); + } + // Rendering to bitmap + if (!fastdeploy::jni::BGR2ARGB888Bitmap(env, argb8888_bitmap, c_vis_im)) { + return JNI_FALSE; + } + return JNI_TRUE; +} + +#ifdef __cplusplus +} +#endif diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/FDModelTag.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/FDModelTag.java new file mode 100644 index 0000000000..19ee71cd38 --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/FDModelTag.java @@ -0,0 +1,8 @@ +package com.baidu.paddle.fastdeploy; + +public enum FDModelTag { + UNKNOWN, + VISION_DETECTION_PICODET, + VISION_DETECTION_PPYOLOE, + VISION_CLASSIFICATION_PPCLS +} diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/FastDeployInitializer.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/FastDeployInitializer.java new file mode 100644 index 0000000000..e69ac3036a --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/FastDeployInitializer.java @@ -0,0 +1,22 @@ +package com.baidu.paddle.fastdeploy; + +/** + * Initializer for FastDeploy. The initialization methods are called by package + * classes only. Public users don't have to call them. Public users can get + * FastDeploy information constants such as JNI lib name in this class. + */ +public class FastDeployInitializer { + /** name of C++ JNI lib */ + public final static String JNI_LIB_NAME = "fastdeploy_jni"; + + /** + * loads the C++ JNI lib. We only call it in our package, so it shouldn't be + * visible to public users. + * + * @return true if initialize successfully. + */ + public static boolean init() { + System.loadLibrary(JNI_LIB_NAME); + return true; + } +} diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/LitePowerMode.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/LitePowerMode.java new file mode 100644 index 0000000000..4e0330da01 --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/LitePowerMode.java @@ -0,0 +1,10 @@ +package com.baidu.paddle.fastdeploy; + +public enum LitePowerMode { + LITE_POWER_HIGH, + LITE_POWER_LOW, + LITE_POWER_FULL, + LITE_POWER_NO_BIND, + LITE_POWER_RAND_HIGH, + LITE_POWER_RAND_LOW +} diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/RuntimeOption.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/RuntimeOption.java new file mode 100644 index 0000000000..471673f596 --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/RuntimeOption.java @@ -0,0 +1,64 @@ +package com.baidu.paddle.fastdeploy; + +public class RuntimeOption { + public int mCpuThreadNum = 1; + public boolean mEnableLiteFp16 = false; + public boolean mEnableRecordTimeOfRuntime = false; + public LitePowerMode mLitePowerMode = LitePowerMode.LITE_POWER_NO_BIND; + public String mLiteOptimizedModelDir = ""; + + public RuntimeOption() { + mCpuThreadNum = 1; + mEnableLiteFp16 = false; + mEnableRecordTimeOfRuntime = false; + mLitePowerMode = LitePowerMode.LITE_POWER_NO_BIND; + mLiteOptimizedModelDir = ""; + } + + public void enableLiteFp16() { + mEnableLiteFp16 = true; + } + + public void disableLiteFP16() { + mEnableLiteFp16 = false; + } + + public void setCpuThreadNum(int threadNum) { + mCpuThreadNum = threadNum; + } + + public void setLitePowerMode(LitePowerMode mode) { + mLitePowerMode = mode; + } + + public void setLitePowerMode(String modeStr) { + mLitePowerMode = parseLitePowerModeFromString(modeStr); + } + + public void setLiteOptimizedModelDir(String modelDir) { + mLiteOptimizedModelDir = modelDir; + } + + public void enableRecordTimeOfRuntime() { + mEnableRecordTimeOfRuntime = true; + } + + // Helpers: parse lite power mode from string + public static LitePowerMode parseLitePowerModeFromString(String modeStr) { + if (modeStr.equalsIgnoreCase("LITE_POWER_HIGH")) { + return LitePowerMode.LITE_POWER_HIGH; + } else if (modeStr.equalsIgnoreCase("LITE_POWER_LOW")) { + return LitePowerMode.LITE_POWER_LOW; + } else if (modeStr.equalsIgnoreCase("LITE_POWER_FULL")) { + return LitePowerMode.LITE_POWER_FULL; + } else if (modeStr.equalsIgnoreCase("LITE_POWER_NO_BIND")) { + return LitePowerMode.LITE_POWER_NO_BIND; + } else if (modeStr.equalsIgnoreCase("LITE_POWER_RAND_HIGH")) { + return LitePowerMode.LITE_POWER_RAND_HIGH; + } else if (modeStr.equalsIgnoreCase("LITE_POWER_RAND_LOW")) { + return LitePowerMode.LITE_POWER_RAND_LOW; + } else { + return LitePowerMode.LITE_POWER_NO_BIND; + } + } +} diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/ClassifyResult.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/ClassifyResult.java new file mode 100644 index 0000000000..7e6e55bd05 --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/ClassifyResult.java @@ -0,0 +1,51 @@ +package com.baidu.paddle.fastdeploy.vision; + +import android.support.annotation.NonNull; + +public class ClassifyResult { + public float[] mScores; // [n] + public int[] mLabelIds; // [n] + public boolean mInitialized = false; + + public ClassifyResult() { + mInitialized = false; + } + + public ClassifyResult(long nativeResultContext) { + mInitialized = copyAllFromNativeContext(nativeResultContext); + } + + public boolean initialized() { + return mInitialized; + } + + private void setScores(@NonNull float[] scoresBuffer) { + if (scoresBuffer.length > 0) { + mScores = scoresBuffer.clone(); + } + } + + private void setLabelIds(@NonNull int[] labelIdsBuffer) { + if (labelIdsBuffer.length > 0) { + mLabelIds = labelIdsBuffer.clone(); + } + } + + private boolean copyAllFromNativeContext(long nativeResultContext) { + if (nativeResultContext == 0) { + return false; + } + setScores(copyScoresFromNative(nativeResultContext)); + setLabelIds(copyLabelIdsFromNative(nativeResultContext)); + // WARN: must release ctx. + return releaseNative(nativeResultContext); + } + + // Fetch native buffers from native context. + private static native float[] copyScoresFromNative(long nativeResultContext); + + private static native int[] copyLabelIdsFromNative(long nativeResultContext); + + private static native boolean releaseNative(long nativeResultContext); + +} diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/DetectionResult.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/DetectionResult.java new file mode 100644 index 0000000000..fefb8a4faf --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/DetectionResult.java @@ -0,0 +1,80 @@ +package com.baidu.paddle.fastdeploy.vision; + +import android.support.annotation.NonNull; + +import com.baidu.paddle.fastdeploy.FastDeployInitializer; + +import java.util.Arrays; + +public class DetectionResult { + // Not support MaskRCNN now. + public float[][] mBoxes; // [n,4] + public float[] mScores; // [n] + public int[] mLabelIds; // [n] + public boolean mInitialized = false; + + public DetectionResult() { + mInitialized = false; + } + + public DetectionResult(long nativeResultContext) { + mInitialized = copyAllFromNativeContext(nativeResultContext); + } + + public boolean initialized() { + return mInitialized; + } + + // Setup results from native buffers. + private boolean copyAllFromNativeContext(long nativeResultContext) { + if (nativeResultContext == 0) { + return false; + } + if (copyBoxesNumFromNative(nativeResultContext) > 0) { + setBoxes(copyBoxesFromNative(nativeResultContext)); + setScores(copyScoresFromNative(nativeResultContext)); + setLabelIds(copyLabelIdsFromNative(nativeResultContext)); + } + // WARN: must release ctx. + return releaseNative(nativeResultContext); + } + + private void setBoxes(@NonNull float[] boxesBuffer) { + int boxesNum = boxesBuffer.length / 4; + if (boxesNum > 0) { + mBoxes = new float[boxesNum][4]; + for (int i = 0; i < boxesNum; ++i) { + mBoxes[i] = Arrays.copyOfRange( + boxesBuffer, i * 4, (i + 1) * 4); + } + } + } + + private void setScores(@NonNull float[] scoresBuffer) { + if (scoresBuffer.length > 0) { + mScores = scoresBuffer.clone(); + } + } + + private void setLabelIds(@NonNull int[] labelIdsBuffer) { + if (labelIdsBuffer.length > 0) { + mLabelIds = labelIdsBuffer.clone(); + } + } + + // Fetch native buffers from native context. + private static native int copyBoxesNumFromNative(long nativeResultContext); + + private static native float[] copyBoxesFromNative(long nativeResultContext); + + private static native float[] copyScoresFromNative(long nativeResultContext); + + private static native int[] copyLabelIdsFromNative(long nativeResultContext); + + private static native boolean releaseNative(long nativeResultContext); + + // Initializes at the beginning. + static { + FastDeployInitializer.init(); + } +} diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/Visualize.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/Visualize.java new file mode 100644 index 0000000000..0c9a13a3b7 --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/Visualize.java @@ -0,0 +1,85 @@ +package com.baidu.paddle.fastdeploy.vision; + +import android.graphics.Bitmap; + +import com.baidu.paddle.fastdeploy.FastDeployInitializer; + + +public class Visualize { + // TODO(qiuyanjun): + // VisClassification, VisSegmentation, VisMatting, VisOcr, ... + + // Visualize DetectionResult without labels + public static boolean visDetection(Bitmap ARGB8888Bitmap, + DetectionResult result) { + return visDetectionNative( + ARGB8888Bitmap, + result.mBoxes, + result.mScores, + result.mLabelIds, + 0.f, 1, 0.5f, + new String[]{}); + } + + public static boolean visDetection(Bitmap ARGB8888Bitmap, + DetectionResult result, + float score_threshold, + int line_size, + float font_size) { + return visDetectionNative( + ARGB8888Bitmap, + result.mBoxes, + result.mScores, + result.mLabelIds, + score_threshold, + line_size, + font_size, + new String[]{}); + } + + // Visualize DetectionResult with labels + public static boolean visDetection(Bitmap ARGB8888Bitmap, + DetectionResult result, + String[] labels) { + return visDetectionNative( + ARGB8888Bitmap, + result.mBoxes, + result.mScores, + result.mLabelIds, + 0.f, 1, 0.5f, + labels); + } + + public static boolean visDetection(Bitmap ARGB8888Bitmap, + DetectionResult result, + float score_threshold, + int line_size, + float font_size, + String[] labels) { + return visDetectionNative( + ARGB8888Bitmap, + result.mBoxes, + result.mScores, + result.mLabelIds, + score_threshold, + line_size, + font_size, + labels); + } + + // VisDetection in native + public static native boolean visDetectionNative(Bitmap ARGB8888Bitmap, + float[][] boxes, + float[] scores, + int[] labelIds, + float score_threshold, + int line_size, + float font_size, + String[] labels); + + + /* Initializes at the beginning */ + static { + FastDeployInitializer.init(); + } +} diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java new file mode 100644 index 0000000000..b4a56e3095 --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java @@ -0,0 +1,172 @@ +package com.baidu.paddle.fastdeploy.vision.classification; + +import android.graphics.Bitmap; + +import com.baidu.paddle.fastdeploy.FastDeployInitializer; +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.vision.ClassifyResult; + +public class PaddleClasModel { + protected long mNativeModelContext = 0; // Context from native. + protected boolean mInitialized = false; + + public PaddleClasModel() { + mInitialized = false; + } + + // Constructor with default runtime option + public PaddleClasModel(String modelFile, + String paramsFile, + String configFile) { + init_(modelFile, paramsFile, configFile, "", new RuntimeOption()); + } + + public PaddleClasModel(String modelFile, + String paramsFile, + String configFile, + String labelFile) { + init_(modelFile, paramsFile, configFile, labelFile, new RuntimeOption()); + } + + // Constructor without label file + public PaddleClasModel(String modelFile, + String paramsFile, + String configFile, + RuntimeOption option) { + init_(modelFile, paramsFile, configFile, "", option); + } + + // Constructor with label file + public PaddleClasModel(String modelFile, + String paramsFile, + String configFile, + String labelFile, + RuntimeOption option) { + init_(modelFile, paramsFile, configFile, labelFile, option); + } + + // Call init manually without label file + public boolean init(String modelFile, + String paramsFile, + String configFile, + RuntimeOption option) { + return init_(modelFile, paramsFile, configFile, "", option); + } + + // Call init manually with label file + public boolean init(String modelFile, + String paramsFile, + String configFile, + String labelFile, + RuntimeOption option) { + return init_(modelFile, paramsFile, configFile, labelFile, option); + } + + + public boolean release() { + mInitialized = false; + if (mNativeModelContext == 0) { + return false; + } + return releaseNative(mNativeModelContext); + } + + public boolean initialized() { + return mInitialized; + } + + // Predict without image saving and bitmap rendering. + public ClassifyResult predict(Bitmap ARGB8888Bitmap) { + if (mNativeModelContext == 0) { + return new ClassifyResult(); + } + // Only support ARGB8888 bitmap in native now. + return new ClassifyResult(predictNative( + mNativeModelContext, ARGB8888Bitmap, false, + "", 0.f, false)); + } + + // Predict with image saving and bitmap rendering (will cost more times) + public ClassifyResult predict(Bitmap ARGB8888Bitmap, + String savedImagePath, + float scoreThreshold) { + // scoreThreshold is for visualizing only. + if (mNativeModelContext == 0) { + return new ClassifyResult(); + } + // Only support ARGB8888 bitmap in native now. + return new ClassifyResult(predictNative( + mNativeModelContext, ARGB8888Bitmap, true, + savedImagePath, scoreThreshold, true)); + } + + // Internal init_ method + private boolean init_(String modelFile, + String paramsFile, + String configFile, + String labelFile, + RuntimeOption option) { + if (!mInitialized) { + mNativeModelContext = bindNative( + modelFile, + paramsFile, + configFile, + option.mCpuThreadNum, + option.mEnableLiteFp16, + option.mLitePowerMode.ordinal(), + option.mLiteOptimizedModelDir, + option.mEnableRecordTimeOfRuntime, labelFile); + if (mNativeModelContext != 0) { + mInitialized = true; + } + return mInitialized; + } else { + // release current native context and bind a new one. + if (release()) { + mNativeModelContext = bindNative( + modelFile, + paramsFile, + configFile, + option.mCpuThreadNum, + option.mEnableLiteFp16, + option.mLitePowerMode.ordinal(), + option.mLiteOptimizedModelDir, + option.mEnableRecordTimeOfRuntime, labelFile); + if (mNativeModelContext != 0) { + mInitialized = true; + } + return mInitialized; + } + return false; + } + } + + + // Bind predictor from native context. + private static native long bindNative(String modelFile, + String paramsFile, + String configFile, + int cpuNumThread, + boolean enableLiteFp16, + int litePowerMode, + String liteOptimizedModelDir, + boolean enableRecordTimeOfRuntime, + String labelFile); + + // Call prediction from native context. + private static native long predictNative(long nativeModelContext, + Bitmap ARGB8888Bitmap, + boolean saved, + String savedImagePath, + float scoreThreshold, + boolean rendering); + + // Release buffers allocated in native context. + private static native boolean releaseNative(long nativeModelContext); + + // Initializes at the beginning. + static { + FastDeployInitializer.init(); + } + +} diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java new file mode 100644 index 0000000000..9729eeb8a4 --- /dev/null +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java @@ -0,0 +1,170 @@ +package com.baidu.paddle.fastdeploy.vision.detection; + +import android.graphics.Bitmap; + +import com.baidu.paddle.fastdeploy.FastDeployInitializer; +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.vision.DetectionResult; + +public class PicoDet { + protected long mNativeModelContext = 0; // Context from native. + protected boolean mInitialized = false; + + public PicoDet() { + mInitialized = false; + } + + // Constructor with default runtime option + public PicoDet(String modelFile, + String paramsFile, + String configFile) { + init_(modelFile, paramsFile, configFile, "", new RuntimeOption()); + } + + public PicoDet(String modelFile, + String paramsFile, + String configFile, + String labelFile) { + init_(modelFile, paramsFile, configFile, labelFile, new RuntimeOption()); + } + + // Constructor without label file + public PicoDet(String modelFile, + String paramsFile, + String configFile, + RuntimeOption option) { + init_(modelFile, paramsFile, configFile, "", option); + } + + // Constructor with label file + public PicoDet(String modelFile, + String paramsFile, + String configFile, + String labelFile, + RuntimeOption option) { + init_(modelFile, paramsFile, configFile, labelFile, option); + } + + // Call init manually without label file + public boolean init(String modelFile, + String paramsFile, + String configFile, + RuntimeOption option) { + return init_(modelFile, paramsFile, configFile, "", option); + } + + // Call init manually with label file + public boolean init(String modelFile, + String paramsFile, + String configFile, + String labelFile, + RuntimeOption option) { + return init_(modelFile, paramsFile, configFile, labelFile, option); + } + + public boolean release() { + mInitialized = false; + if (mNativeModelContext == 0) { + return false; + } + return releaseNative(mNativeModelContext); + } + + public boolean initialized() { + return mInitialized; + } + + // Predict without image saving and bitmap rendering. + public DetectionResult predict(Bitmap ARGB8888Bitmap) { + if (mNativeModelContext == 0) { + return new DetectionResult(); + } + // Only support ARGB8888 bitmap in native now. + return new DetectionResult(predictNative( + mNativeModelContext, ARGB8888Bitmap, false, + "", 0.f, false)); + } + + // Predict with image saving and bitmap rendering (will cost more times) + public DetectionResult predict(Bitmap ARGB8888Bitmap, + String savedImagePath, + float scoreThreshold) { + // scoreThreshold is for visualizing only. + if (mNativeModelContext == 0) { + return new DetectionResult(); + } + // Only support ARGB8888 bitmap in native now. + return new DetectionResult(predictNative( + mNativeModelContext, ARGB8888Bitmap, true, + savedImagePath, scoreThreshold, true)); + } + + + private boolean init_(String modelFile, + String paramsFile, + String configFile, + String labelFile, + RuntimeOption option) { + if (!mInitialized) { + mNativeModelContext = bindNative( + modelFile, + paramsFile, + configFile, + option.mCpuThreadNum, + option.mEnableLiteFp16, + option.mLitePowerMode.ordinal(), + option.mLiteOptimizedModelDir, + option.mEnableRecordTimeOfRuntime, labelFile); + if (mNativeModelContext != 0) { + mInitialized = true; + } + return mInitialized; + } else { + // release current native context and bind a new one. + if (release()) { + mNativeModelContext = bindNative( + modelFile, + paramsFile, + configFile, + option.mCpuThreadNum, + option.mEnableLiteFp16, + option.mLitePowerMode.ordinal(), + option.mLiteOptimizedModelDir, + option.mEnableRecordTimeOfRuntime, labelFile); + if (mNativeModelContext != 0) { + mInitialized = true; + } + return mInitialized; + } + return false; + } + } + + // Bind predictor from native context. + private static native long bindNative(String modelFile, + String paramsFile, + String configFile, + int cpuNumThread, + boolean enableLiteFp16, + int litePowerMode, + String liteOptimizedModelDir, + boolean enableRecordTimeOfRuntime, + String labelFile); + + // Call prediction from native context. + private static native long predictNative(long nativeModelContext, + Bitmap ARGB8888Bitmap, + boolean saved, + String savedImagePath, + float scoreThreshold, + boolean rendering); + + // Release buffers allocated in native context. + private static native boolean releaseNative(long nativeModelContext); + + // Initializes at the beginning. + static { + FastDeployInitializer.init(); + } +} + diff --git a/java/android/fastdeploy/src/test/java/com/baidu/paddle/fastdeploy/ExampleUnitTest.java b/java/android/fastdeploy/src/test/java/com/baidu/paddle/fastdeploy/ExampleUnitTest.java new file mode 100644 index 0000000000..3e3fa8a1f4 --- /dev/null +++ b/java/android/fastdeploy/src/test/java/com/baidu/paddle/fastdeploy/ExampleUnitTest.java @@ -0,0 +1,17 @@ +package com.baidu.paddle.fastdeploy; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Example local unit test, which will execute on the development machine (host). + * + * @see Testing documentation + */ +public class ExampleUnitTest { + @Test + public void addition_isCorrect() { + assertEquals(4, 2 + 2); + } +} \ No newline at end of file diff --git a/java/android/gradle.properties b/java/android/gradle.properties new file mode 100644 index 0000000000..ae995d47cc --- /dev/null +++ b/java/android/gradle.properties @@ -0,0 +1,13 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx3096m +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true diff --git a/java/android/gradle/wrapper/gradle-wrapper.jar b/java/android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000..e708b1c023 Binary files /dev/null and b/java/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/java/android/gradle/wrapper/gradle-wrapper.properties b/java/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000..7855fafe49 --- /dev/null +++ b/java/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Sat Oct 08 17:24:34 CST 2022 +distributionBase=GRADLE_USER_HOME +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionPath=wrapper/dists +zipStorePath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME diff --git a/java/android/gradlew b/java/android/gradlew new file mode 100644 index 0000000000..4f906e0c81 --- /dev/null +++ b/java/android/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/java/android/gradlew.bat b/java/android/gradlew.bat new file mode 100644 index 0000000000..ac1b06f938 --- /dev/null +++ b/java/android/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/java/android/local.properties b/java/android/local.properties new file mode 100644 index 0000000000..fbfca12ef8 --- /dev/null +++ b/java/android/local.properties @@ -0,0 +1,8 @@ +## This file must *NOT* be checked into Version Control Systems, +# as it contains information specific to your local configuration. +# +# Location of the SDK. This is only used by Gradle. +# For customization when using a Version Control System, please read the +# header note. +#Thu Oct 20 16:50:08 CST 2022 +sdk.dir=/Users/qiuyanjun/Library/Android/sdk diff --git a/java/android/settings.gradle b/java/android/settings.gradle new file mode 100644 index 0000000000..2a764b812b --- /dev/null +++ b/java/android/settings.gradle @@ -0,0 +1,2 @@ +include ':app' +include ':fastdeploy'