From 6723d1d2d998a6884ffee7c48a49b47c4a42fb5b Mon Sep 17 00:00:00 2001 From: jiangjiajun Date: Tue, 20 Sep 2022 12:10:59 +0000 Subject: [PATCH 1/2] Add Paddle-Lite backend support --- CMakeLists.txt | 12 +- FastDeploy.cmake.in | 6 + cmake/paddlelite.cmake | 72 +++++++++ .../paddledetection/cpp/CMakeLists.txt | 2 + fastdeploy/backends/lite/lite_backend.cc | 143 ++++++++++++++++++ fastdeploy/backends/lite/lite_backend.h | 59 ++++++++ fastdeploy/core/allocate.cc | 1 - fastdeploy/fastdeploy_runtime.cc | 51 ++++++- fastdeploy/fastdeploy_runtime.h | 7 +- fastdeploy/pybind/fastdeploy_runtime.cc | 4 +- fastdeploy/vision/detection/ppdet/picodet.cc | 2 +- fastdeploy/vision/detection/ppdet/ppyoloe.cc | 2 + fastdeploy/vision/detection/ppdet/ppyoloe.h | 2 +- 13 files changed, 350 insertions(+), 13 deletions(-) create mode 100644 cmake/paddlelite.cmake create mode 100644 fastdeploy/backends/lite/lite_backend.cc create mode 100644 fastdeploy/backends/lite/lite_backend.h diff --git a/CMakeLists.txt b/CMakeLists.txt index cd71e143..6d422717 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,8 @@ option(WITH_GPU "Whether WITH_GPU=ON, will enable onnxruntime-gpu/paddle-infernc option(ENABLE_ORT_BACKEND "Whether to enable onnxruntime backend." OFF) option(ENABLE_TRT_BACKEND "Whether to enable tensorrt backend." OFF) option(ENABLE_PADDLE_BACKEND "Whether to enable paddle backend." OFF) -option(ENABLE_OPENVINO_BACKEND "Whether to enable paddle backend." OFF) +option(ENABLE_OPENVINO_BACKEND "Whether to enable paddle inference backend." OFF) +option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF) option(CUDA_DIRECTORY "If build tensorrt backend, need to define path of cuda library.") option(TRT_DIRECTORY "If build tensorrt backend, need to define path of tensorrt library.") option(ENABLE_VISION "Whether to enable vision models usage." OFF) @@ -120,10 +121,11 @@ file(GLOB_RECURSE DEPLOY_ORT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastde file(GLOB_RECURSE DEPLOY_PADDLE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/paddle/*.cc) file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/tensorrt/*.cpp) file(GLOB_RECURSE DEPLOY_OPENVINO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/openvino/*.cc) +file(GLOB_RECURSE DEPLOY_LITE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/backends/lite/*.cc) file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/vision/*.cc) file(GLOB_RECURSE DEPLOY_TEXT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/text/*.cc) file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/pybind/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/*_pybind.cc) -list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS}) +list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS} ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS}) set(DEPEND_LIBS "") @@ -147,6 +149,12 @@ if(ENABLE_ORT_BACKEND) list(APPEND DEPEND_LIBS external_onnxruntime) endif() +if(ENABLE_LITE_BACKEND) + add_definitions(-DENABLE_LITE_BACKEND) + include(${PROJECT_SOURCE_DIR}/cmake/paddlelite.cmake) + list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_LITE_SRCS}) +endif() + if(ENABLE_PADDLE_BACKEND) add_definitions(-DENABLE_PADDLE_BACKEND) list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_PADDLE_SRCS}) diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index 3ec9d2a6..1afabfb0 100644 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -4,6 +4,7 @@ set(WITH_GPU @WITH_GPU@) set(ENABLE_ORT_BACKEND @ENABLE_ORT_BACKEND@) set(ENABLE_PADDLE_BACKEND @ENABLE_PADDLE_BACKEND@) set(ENABLE_OPENVINO_BACKEND @ENABLE_OPENVINO_BACKEND@) +set(ENABLE_LITE_BACKEND @ENABLE_LITE_BACKEND@) set(PADDLEINFERENCE_VERSION @PADDLEINFERENCE_VERSION@) set(OPENVINO_VERSION @OPENVINO_VERSION@) set(ENABLE_TRT_BACKEND @ENABLE_TRT_BACKEND@) @@ -52,6 +53,11 @@ if(ENABLE_OPENVINO_BACKEND) list(APPEND FASTDEPLOY_LIBS ${OPENVINO_LIB}) endif() +if(ENABLE_LITE_BACKEND) + find_library(LITE_LIB paddle_full_api_shared ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddlelite/lib/ NO_DEFAULT_PATH) + list(APPEND FASTDEPLOY_LIBS ${LITE_LIB}) +endif() + if(WITH_GPU) if (NOT CUDA_DIRECTORY) set(CUDA_DIRECTORY "/usr/local/cuda") diff --git a/cmake/paddlelite.cmake b/cmake/paddlelite.cmake new file mode 100644 index 00000000..8cd14315 --- /dev/null +++ b/cmake/paddlelite.cmake @@ -0,0 +1,72 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(ExternalProject) + +set(PADDLELITE_PROJECT "extern_paddlelite") +set(PADDLELITE_PREFIX_DIR ${THIRD_PARTY_PATH}/paddlelite) +set(PADDLELITE_SOURCE_DIR + ${THIRD_PARTY_PATH}/paddlelite/src/${PADDLELITE_PROJECT}) +set(PADDLELITE_INSTALL_DIR ${THIRD_PARTY_PATH}/install/paddlelite) +set(PADDLELITE_INC_DIR + "${PADDLELITE_INSTALL_DIR}/include" + CACHE PATH "paddlelite include directory." FORCE) +set(PADDLELITE_LIB_DIR + "${PADDLELITE_INSTALL_DIR}/lib" + CACHE PATH "paddlelite lib directory." FORCE) +set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${PADDLELITE_LIB_DIR}") + +#set(PADDLELITE_URL "https://github.com/PaddlePaddle/Paddle-Lite/releases/download/v2.11/inference_lite_lib.armlinux.armv8.gcc.with_extra.with_cv.tar.gz") +set(PADDLELITE_URL "https://bj.bcebos.com/fastdeploy/third_libs/lite-linux-arm64-20220920.tgz") + +if(WIN32) + message(FATAL_ERROR "Doesn't support windows platform with backend Paddle-Lite.") +elseif(APPLE) + message(FATAL_ERROR "Doesn't support Mac OSX platform with backend Paddle-Lite.") +else() + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") + set(PADDLELITE_URL "https://bj.bcebos.com/fastdeploy/third_libs/lite-linux-arm64-20220920.tgz") + else() + message(FATAL_ERROR "Only support Linux aarch64 now, x64 is not supported with backend Paddle-Lite.") + endif() +endif() + +include_directories(${PADDLELITE_INC_DIR}) + +if(WIN32) +elseif(APPLE) +else() + set(PADDLELITE_LIB "${PADDLELITE_INSTALL_DIR}/lib/libpaddle_full_api_shared.so") +endif() + +ExternalProject_Add( + ${PADDLELITE_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + URL ${PADDLELITE_URL} + PREFIX ${PADDLELITE_PREFIX_DIR} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} -E remove_directory ${PADDLELITE_INSTALL_DIR} && + ${CMAKE_COMMAND} -E make_directory ${PADDLELITE_INSTALL_DIR} && + ${CMAKE_COMMAND} -E rename ${PADDLELITE_SOURCE_DIR}/lib/ ${PADDLELITE_INSTALL_DIR}/lib && + ${CMAKE_COMMAND} -E copy_directory ${PADDLELITE_SOURCE_DIR}/include + ${PADDLELITE_INC_DIR} + BUILD_BYPRODUCTS ${PADDLELITE_LIB}) + +add_library(external_paddle_lite STATIC IMPORTED GLOBAL) +set_property(TARGET external_paddle_lite PROPERTY IMPORTED_LOCATION ${PADDLELITE_LIB}) +add_dependencies(external_paddle_lite ${PADDLELITE_PROJECT}) diff --git a/examples/vision/detection/paddledetection/cpp/CMakeLists.txt b/examples/vision/detection/paddledetection/cpp/CMakeLists.txt index de311873..a132c930 100644 --- a/examples/vision/detection/paddledetection/cpp/CMakeLists.txt +++ b/examples/vision/detection/paddledetection/cpp/CMakeLists.txt @@ -29,3 +29,5 @@ target_link_libraries(infer_ppyolo_demo ${FASTDEPLOY_LIBS}) add_executable(infer_mask_rcnn_demo ${PROJECT_SOURCE_DIR}/infer_mask_rcnn.cc) target_link_libraries(infer_mask_rcnn_demo ${FASTDEPLOY_LIBS}) + +message("???????? ${FASTDEPLOY_LIBS}") diff --git a/fastdeploy/backends/lite/lite_backend.cc b/fastdeploy/backends/lite/lite_backend.cc new file mode 100644 index 00000000..25a79dbf --- /dev/null +++ b/fastdeploy/backends/lite/lite_backend.cc @@ -0,0 +1,143 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/backends/lite/lite_backend.h" +#include + +namespace fastdeploy { + +// Convert data type from paddle lite to fastdeploy +FDDataType LiteDataTypeToFD(const paddle::lite_api::PrecisionType& dtype) { + if (dtype == paddle::lite_api::PrecisionType::kFloat) { + return FDDataType::FP32; + } else if (dtype == paddle::lite_api::PrecisionType::kInt8) { + return FDDataType::INT8; + } else if (dtype == paddle::lite_api::PrecisionType::kInt32) { + return FDDataType::INT32; + } else if (dtype == paddle::lite_api::PrecisionType::kInt64) { + return FDDataType::INT64; + } else if (dtype == paddle::lite_api::PrecisionType::kInt16) { + return FDDataType::INT16; + } else if (dtype == paddle::lite_api::PrecisionType::kUInt8) { + return FDDataType::UINT8; + } else if (dtype == paddle::lite_api::PrecisionType::kFP64) { + return FDDataType::FP64; + } + FDASSERT(false, "Unexpected data type of %d.", dtype); + return FDDataType::FP32; +} + +void LiteBackend::BuildOption(const LiteBackendOption& option) { + // do nothing + return; +} + +bool LiteBackend::InitFromPaddle(const std::string& model_file, + const std::string& params_file, + const LiteBackendOption& option) { + if (initialized_) { + FDERROR << "LiteBackend is already initialized, cannot initialize again." + << std::endl; + return false; + } + + FDERROR << "????????00000" << std::endl; + BuildOption(option); + config_.set_model_file(model_file); + config_.set_param_file(params_file); + FDERROR << "????????00001" << std::endl; + predictor_ = paddle::lite_api::CreatePaddlePredictor(config_); + FDERROR << "????????00002" << std::endl; + + inputs_desc_.clear(); + outputs_desc_.clear(); + inputs_order_.clear(); + std::vector input_names = predictor_->GetInputNames(); + std::vector output_names = predictor_->GetOutputNames(); + for (size_t i = 0; i < input_names.size(); ++i) { + inputs_order_[input_names[i]] = i; + TensorInfo info; + auto tensor = predictor_->GetInput(i); + auto shape = tensor->shape(); + info.shape.assign(shape.begin(), shape.end()); + info.name = input_names[i]; + info.dtype = LiteDataTypeToFD(tensor->precision()); + inputs_desc_.emplace_back(info); + } + for (size_t i = 0; i < output_names.size(); ++i) { + TensorInfo info; + auto tensor = predictor_->GetOutput(i); + auto shape = tensor->shape(); + info.shape.assign(shape.begin(), shape.end()); + info.name = output_names[i]; + info.dtype = LiteDataTypeToFD(tensor->precision()); + outputs_desc_.emplace_back(info); + } + + initialized_ = true; + return true; +} + +TensorInfo LiteBackend::GetInputInfo(int index) { + FDASSERT(index < NumInputs(), + "The index: %d should less than the number of inputs: %d.", index, + NumInputs()); + return inputs_desc_[index]; +} + +std::vector LiteBackend::GetInputInfos() { return inputs_desc_; } + +TensorInfo LiteBackend::GetOutputInfo(int index) { + FDASSERT(index < NumOutputs(), + "The index: %d should less than the number of outputs %d.", index, + NumOutputs()); + return outputs_desc_[index]; +} + +std::vector LiteBackend::GetOutputInfos() { + return outputs_desc_; +} + +bool LiteBackend::Infer(std::vector& inputs, + std::vector* outputs) { + if (inputs.size() != inputs_desc_.size()) { + FDERROR << "[LiteBackend] Size of inputs(" << inputs.size() + << ") should keep same with the inputs of this model(" + << inputs_desc_.size() << ")." << std::endl; + return false; + } + + for (size_t i = 0; i < inputs.size(); ++i) { + auto iter = inputs_order_.find(inputs[i].name); + if (iter == inputs_order_.end()) { + FDERROR << "Cannot find input with name:" << inputs[i].name << " in loaded model." << std::endl; + return false; + } + auto tensor = predictor_->GetInput(iter->second); + tensor->Resize(inputs[i].shape); + tensor->ShareExternalMemory(const_cast(inputs[i].CpuData()), inputs[i].Nbytes(), paddle::lite_api::TargetType::kARM); + } + + predictor_->Run(); + + outputs->resize(outputs_desc_.size()); + for (size_t i = 0; i < outputs_desc_.size(); ++i) { + auto tensor = predictor_->GetOutput(i); + (*outputs)[i].Resize(tensor->shape(), outputs_desc_[i].dtype, outputs_desc_[i].name); + memcpy((*outputs)[i].MutableData(), tensor->data(), (*outputs)[i].Nbytes()); + } + return true; +} + +} // namespace fastdeploy diff --git a/fastdeploy/backends/lite/lite_backend.h b/fastdeploy/backends/lite/lite_backend.h new file mode 100644 index 00000000..978982b3 --- /dev/null +++ b/fastdeploy/backends/lite/lite_backend.h @@ -0,0 +1,59 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "fastdeploy/backends/backend.h" +#include "paddle_api.h" // NOLINT + +namespace fastdeploy { + +struct LiteBackendOption { +}; + +// Convert data type from paddle lite to fastdeploy +FDDataType LiteDataTypeToFD(const paddle::lite_api::PrecisionType& dtype); + +class LiteBackend : public BaseBackend { + public: + LiteBackend() {} + virtual ~LiteBackend() = default; + void BuildOption(const LiteBackendOption& option); + + bool InitFromPaddle(const std::string& model_file, const std::string& params_file, const LiteBackendOption& option = LiteBackendOption()); + + bool Infer(std::vector& inputs, std::vector* outputs); + + int NumInputs() const { return inputs_desc_.size(); } + + int NumOutputs() const { return outputs_desc_.size(); } + + TensorInfo GetInputInfo(int index); + TensorInfo GetOutputInfo(int index); + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; + + private: + paddle::lite_api::CxxConfig config_; + std::shared_ptr predictor_; + std::vector inputs_desc_; + std::vector outputs_desc_; + std::map inputs_order_; +}; +} // namespace fastdeploy diff --git a/fastdeploy/core/allocate.cc b/fastdeploy/core/allocate.cc index 0615ee46..44a9e774 100644 --- a/fastdeploy/core/allocate.cc +++ b/fastdeploy/core/allocate.cc @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#pragma once #ifdef WITH_GPU #include diff --git a/fastdeploy/fastdeploy_runtime.cc b/fastdeploy/fastdeploy_runtime.cc index 3c119f7a..92ecc242 100644 --- a/fastdeploy/fastdeploy_runtime.cc +++ b/fastdeploy/fastdeploy_runtime.cc @@ -33,6 +33,10 @@ #include "fastdeploy/backends/openvino/ov_backend.h" #endif +#ifdef ENABLE_LITE_BACKEND +#include "fastdeploy/backends/lite/lite_backend.h" +#endif + namespace fastdeploy { std::vector GetAvailableBackends() { @@ -48,6 +52,9 @@ std::vector GetAvailableBackends() { #endif #ifdef ENABLE_OPENVINO_BACKEND backends.push_back(Backend::OPENVINO); +#endif +#ifdef ENABLE_LITE_BACKEND + backends.push_back(Backend::LITE); #endif return backends; } @@ -71,6 +78,8 @@ std::string Str(const Backend& b) { return "Backend::PDINFER"; } else if (b == Backend::OPENVINO) { return "Backend::OPENVINO"; + } else if (b == Backend::LITE) { + return "Backend::LITE"; } return "UNKNOWN-Backend"; } @@ -114,11 +123,11 @@ bool CheckModelFormat(const std::string& model_file, Frontend GuessModelFormat(const std::string& model_file) { if (model_file.size() > 8 && model_file.substr(model_file.size() - 8, 8) == ".pdmodel") { - FDLogger() << "Model Format: PaddlePaddle." << std::endl; + FDINFO << "Model Format: PaddlePaddle." << std::endl; return Frontend::PADDLE; } else if (model_file.size() > 5 && model_file.substr(model_file.size() - 5, 5) == ".onnx") { - FDLogger() << "Model Format: ONNX." << std::endl; + FDINFO << "Model Format: ONNX." << std::endl; return Frontend::ONNX; } @@ -194,6 +203,15 @@ void RuntimeOption::UseOpenVINOBackend() { FDASSERT(false, "The FastDeploy didn't compile with OpenVINO."); #endif } + +void RuntimeOption::UseLiteBackend() { +#ifdef ENABLE_LITE_BACKEND + backend = Backend::LITE; +#else + FDASSERT(false, "The FastDeploy didn't compile with Paddle Lite."); +#endif +} + void RuntimeOption::EnablePaddleMKLDNN() { pd_enable_mkldnn = true; } void RuntimeOption::DisablePaddleMKLDNN() { pd_enable_mkldnn = false; } @@ -262,12 +280,12 @@ bool Runtime::Init(const RuntimeOption& _option) { FDASSERT(option.device == Device::CPU || option.device == Device::GPU, "Backend::ORT only supports Device::CPU/Device::GPU."); CreateOrtBackend(); - FDINFO << "Runtime initialized with Backend::ORT in device " << Str(option.device) << "." << std::endl; + FDINFO << "Runtime initialized with Backend::ORT in " << Str(option.device) << "." << std::endl; } else if (option.backend == Backend::TRT) { FDASSERT(option.device == Device::GPU, "Backend::TRT only supports Device::GPU."); CreateTrtBackend(); - FDINFO << "Runtime initialized with Backend::TRT in device " << Str(option.device) << "." << std::endl; + FDINFO << "Runtime initialized with Backend::TRT in " << Str(option.device) << "." << std::endl; } else if (option.backend == Backend::PDINFER) { FDASSERT(option.device == Device::CPU || option.device == Device::GPU, "Backend::TRT only supports Device::CPU/Device::GPU."); @@ -275,12 +293,16 @@ bool Runtime::Init(const RuntimeOption& _option) { option.model_format == Frontend::PADDLE, "Backend::PDINFER only supports model format of Frontend::PADDLE."); CreatePaddleBackend(); - FDINFO << "Runtime initialized with Backend::PDINFER in device " << Str(option.device) << "." << std::endl; + FDINFO << "Runtime initialized with Backend::PDINFER in " << Str(option.device) << "." << std::endl; } else if (option.backend == Backend::OPENVINO) { FDASSERT(option.device == Device::CPU, "Backend::OPENVINO only supports Device::CPU"); CreateOpenVINOBackend(); - FDINFO << "Runtime initialized with Backend::OPENVINO in device " << Str(option.device) << "." << std::endl; + FDINFO << "Runtime initialized with Backend::OPENVINO in " << Str(option.device) << "." << std::endl; + } else if (option.backend == Backend::LITE) { + FDASSERT(option.device == Device::CPU, "Backend::LITE only supports Device::CPU"); + CreateLiteBackend(); + FDINFO << "Runtime initialized with Backend::LITE in " << Str(option.device) << "." << std::endl; } else { FDERROR << "Runtime only support " "Backend::ORT/Backend::TRT/Backend::PDINFER as backend now." @@ -433,4 +455,21 @@ void Runtime::CreateTrtBackend() { "ENABLE_TRT_BACKEND=ON."); #endif } + +void Runtime::CreateLiteBackend() { +#ifdef ENABLE_LITE_BACKEND + auto lite_option = LiteBackendOption(); + FDASSERT(option.model_format == Frontend::PADDLE, + "LiteBackend only support model format of Frontend::PADDLE"); + backend_ = utils::make_unique(); + auto casted_backend = dynamic_cast(backend_.get()); + FDASSERT(casted_backend->InitFromPaddle(option.model_file, option.params_file, lite_option), + "Load model from nb file failed while initializing LiteBackend."); +#else + FDASSERT(false, + "LiteBackend is not available, please compiled with " + "ENABLE_LITE_BACKEND=ON."); +#endif +} + } // namespace fastdeploy diff --git a/fastdeploy/fastdeploy_runtime.h b/fastdeploy/fastdeploy_runtime.h index 58981687..c2a5e664 100644 --- a/fastdeploy/fastdeploy_runtime.h +++ b/fastdeploy/fastdeploy_runtime.h @@ -21,7 +21,7 @@ namespace fastdeploy { -enum FASTDEPLOY_DECL Backend { UNKNOWN, ORT, TRT, PDINFER, OPENVINO }; +enum FASTDEPLOY_DECL Backend { UNKNOWN, ORT, TRT, PDINFER, OPENVINO, LITE }; // AUTOREC will according to the name of model file // to decide which Frontend is enum FASTDEPLOY_DECL Frontend { AUTOREC, PADDLE, ONNX }; @@ -66,6 +66,9 @@ struct FASTDEPLOY_DECL RuntimeOption { // use openvino backend void UseOpenVINOBackend(); + // use paddle lite backend + void UseLiteBackend(); + // enable mkldnn while use paddle inference in CPU void EnablePaddleMKLDNN(); // disable mkldnn while use paddle inference in CPU @@ -161,6 +164,8 @@ struct FASTDEPLOY_DECL Runtime { void CreateOpenVINOBackend(); + void CreateLiteBackend(); + int NumInputs() { return backend_->NumInputs(); } int NumOutputs() { return backend_->NumOutputs(); } TensorInfo GetInputInfo(int index); diff --git a/fastdeploy/pybind/fastdeploy_runtime.cc b/fastdeploy/pybind/fastdeploy_runtime.cc index a5924b0a..ad4715a0 100644 --- a/fastdeploy/pybind/fastdeploy_runtime.cc +++ b/fastdeploy/pybind/fastdeploy_runtime.cc @@ -27,6 +27,7 @@ void BindRuntime(pybind11::module& m) { .def("use_ort_backend", &RuntimeOption::UseOrtBackend) .def("use_trt_backend", &RuntimeOption::UseTrtBackend) .def("use_openvino_backend", &RuntimeOption::UseOpenVINOBackend) + .def("use_lite_backend", &RuntimeOption::UseLiteBackend) .def("enable_paddle_mkldnn", &RuntimeOption::EnablePaddleMKLDNN) .def("disable_paddle_mkldnn", &RuntimeOption::DisablePaddleMKLDNN) .def("enable_paddle_log_info", &RuntimeOption::EnablePaddleLogInfo) @@ -109,7 +110,8 @@ void BindRuntime(pybind11::module& m) { .value("UNKOWN", Backend::UNKNOWN) .value("ORT", Backend::ORT) .value("TRT", Backend::TRT) - .value("PDINFER", Backend::PDINFER); + .value("PDINFER", Backend::PDINFER) + .value("LITE", Backend::LITE); pybind11::enum_(m, "Frontend", pybind11::arithmetic(), "Frontend for inference.") .value("PADDLE", Frontend::PADDLE) diff --git a/fastdeploy/vision/detection/ppdet/picodet.cc b/fastdeploy/vision/detection/ppdet/picodet.cc index bae51b40..3b651ccf 100644 --- a/fastdeploy/vision/detection/ppdet/picodet.cc +++ b/fastdeploy/vision/detection/ppdet/picodet.cc @@ -24,7 +24,7 @@ PicoDet::PicoDet(const std::string& model_file, const std::string& params_file, const RuntimeOption& custom_option, const Frontend& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; diff --git a/fastdeploy/vision/detection/ppdet/ppyoloe.cc b/fastdeploy/vision/detection/ppdet/ppyoloe.cc index 69aacb3f..8f95a1ac 100644 --- a/fastdeploy/vision/detection/ppdet/ppyoloe.cc +++ b/fastdeploy/vision/detection/ppdet/ppyoloe.cc @@ -24,6 +24,7 @@ PPYOLOE::PPYOLOE(const std::string& model_file, const std::string& params_file, } void PPYOLOE::GetNmsInfo() { +#ifdef ENABLE_PADDLE_FRONTEND if (runtime_option.model_format == Frontend::PADDLE) { std::string contents; if (!ReadBinaryFromFile(runtime_option.model_file, &contents)) { @@ -41,6 +42,7 @@ void PPYOLOE::GetNmsInfo() { normalized = reader.nms_params.normalized; } } +#endif } bool PPYOLOE::Initialize() { diff --git a/fastdeploy/vision/detection/ppdet/ppyoloe.h b/fastdeploy/vision/detection/ppdet/ppyoloe.h index 2d8cca99..26894759 100644 --- a/fastdeploy/vision/detection/ppdet/ppyoloe.h +++ b/fastdeploy/vision/detection/ppdet/ppyoloe.h @@ -56,7 +56,7 @@ class FASTDEPLOY_DECL PPYOLOE : public FastDeployModel { float score_threshold = 0.01; int64_t nms_top_k = 10000; bool normalized = true; - bool has_nms_ = false; + bool has_nms_ = true; // This function will used to check if this model contains multiclass_nms // and get parameters from the operator -- Gitee From d38c9f40e5116f9fb8ee1dc5ffa275bb565394dc Mon Sep 17 00:00:00 2001 From: jiangjiajun Date: Tue, 20 Sep 2022 21:49:19 +0800 Subject: [PATCH 2/2] fix error for lite --- fastdeploy/backends/lite/lite_backend.cc | 10 ++++------ fastdeploy/vision/classification/ppcls/model.cc | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/fastdeploy/backends/lite/lite_backend.cc b/fastdeploy/backends/lite/lite_backend.cc index 25a79dbf..29113283 100644 --- a/fastdeploy/backends/lite/lite_backend.cc +++ b/fastdeploy/backends/lite/lite_backend.cc @@ -39,8 +39,9 @@ FDDataType LiteDataTypeToFD(const paddle::lite_api::PrecisionType& dtype) { } void LiteBackend::BuildOption(const LiteBackendOption& option) { - // do nothing - return; + std::vector valid_places; + valid_places.push_back(paddle::lite_api::Place{TARGET(kARM), PRECISION(kFloat)}); + config_.set_valid_places(valid_places); } bool LiteBackend::InitFromPaddle(const std::string& model_file, @@ -52,13 +53,10 @@ bool LiteBackend::InitFromPaddle(const std::string& model_file, return false; } - FDERROR << "????????00000" << std::endl; - BuildOption(option); config_.set_model_file(model_file); config_.set_param_file(params_file); - FDERROR << "????????00001" << std::endl; + BuildOption(option); predictor_ = paddle::lite_api::CreatePaddlePredictor(config_); - FDERROR << "????????00002" << std::endl; inputs_desc_.clear(); outputs_desc_.clear(); diff --git a/fastdeploy/vision/classification/ppcls/model.cc b/fastdeploy/vision/classification/ppcls/model.cc index a738ac29..fb02125f 100644 --- a/fastdeploy/vision/classification/ppcls/model.cc +++ b/fastdeploy/vision/classification/ppcls/model.cc @@ -26,7 +26,7 @@ PaddleClasModel::PaddleClasModel(const std::string& model_file, const RuntimeOption& custom_option, const Frontend& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::ORT, Backend::OPENVINO, Backend::PDINFER}; + valid_cpu_backends = {Backend::ORT, Backend::OPENVINO, Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; -- Gitee