diff --git a/MLModelRunner/CMakeLists.txt b/MLModelRunner/CMakeLists.txt index 1d3c171..396a58a 100755 --- a/MLModelRunner/CMakeLists.txt +++ b/MLModelRunner/CMakeLists.txt @@ -4,6 +4,7 @@ if(NOT PROTOS_DIRECTORY STREQUAL "") add_subdirectory(gRPCModelRunner) endif() add_subdirectory(ONNXModelRunner) +add_subdirectory(PTModelRunner) add_subdirectory(C) # # For up-to-date instructions for installing the TFLite dependency, refer to @@ -44,13 +45,8 @@ else() add_library(ModelRunnerLib OBJECT MLModelRunner.cpp PipeModelRunner.cpp) endif(LLVM_MLBRIDGE) -target_link_libraries(ModelRunnerLib PUBLIC ModelRunnerUtils ONNXModelRunnerLib) +target_link_libraries(ModelRunnerLib PUBLIC ModelRunnerUtils ONNXModelRunnerLib PTModelRunnerLib) if(NOT PROTOS_DIRECTORY STREQUAL "") target_link_libraries(ModelRunnerLib PUBLIC gRPCModelRunnerLib) -endif() -set_property(TARGET ModelRunnerLib PROPERTY POSITION_INDEPENDENT_CODE 1) - -find_package(Torch REQUIRED) -target_link_libraries(ModelRunnerLib PRIVATE ${TORCH_LIBRARIES}) -target_compile_options (ModelRunnerLib PRIVATE -fexceptions) +endif() \ No newline at end of file diff --git a/MLModelRunner/PTModelRunner/CMakeLists.txt b/MLModelRunner/PTModelRunner/CMakeLists.txt new file mode 100755 index 0000000..8537455 --- /dev/null +++ b/MLModelRunner/PTModelRunner/CMakeLists.txt @@ -0,0 +1,13 @@ +if(LLVM_MLBRIDGE) + add_llvm_library(PTModelRunnerLib PTModelRunner.cpp) +else() + add_library(PTModelRunnerLib OBJECT PTModelRunner.cpp) +endif(LLVM_MLBRIDGE) + +find_package(Torch REQUIRED) +target_link_libraries(PTModelRunnerLib PRIVATE ${TORCH_LIBRARIES}) +target_compile_options (PTModelRunnerLib PRIVATE -fexceptions) + +# message("HERE FIND TORCH ${TORCH_INCLUDE_DIRS}") +# find_package(Torch REQUIRED) +target_include_directories(PTModelRunnerLib PRIVATE ${TORCH_INCLUDE_DIRS}) \ No newline at end of file diff --git a/MLModelRunner/PTModelRunner/PTModelRunner.cpp b/MLModelRunner/PTModelRunner/PTModelRunner.cpp new file mode 100644 index 0000000..bba3e84 --- /dev/null +++ b/MLModelRunner/PTModelRunner/PTModelRunner.cpp @@ -0,0 +1,70 @@ +//=== PTModelRunner.cpp - PTModelRunner Implementation ---*- C++ -*-===// +// +// Part of the MLCompilerBridge Project +// +//===------------------===// + +#include "MLModelRunner/PTModelRunner.h" + +#include "MLModelRunner/MLModelRunner.h" +#include "SerDes/TensorSpec.h" +// #include "SerDes/baseSerDes.h" +#include "SerDes/pytorchSerDes.h" +#include "llvm/Support/ErrorHandling.h" +#include +#include // or model_container_runner_cuda.h for CUDA + +#include +#include + +using TensorVec = std::vector; + +namespace MLBridge +{ + + PTModelRunner::PTModelRunner(const std::string &modelPath, llvm::LLVMContext &Ctx) + : MLModelRunner(MLModelRunner::Kind::PTAOT, BaseSerDes::Kind::Pytorch, &Ctx) + { + this->SerDes = new PytorchSerDes(); + + c10::InferenceMode mode; + this->CompiledModel = new torch::inductor::AOTIModelContainerRunnerCpu(modelPath); + } + + + + void *PTModelRunner::evaluateUntyped() + { + + if ((*reinterpret_cast(this->SerDes->getRequest())).empty()) + { + llvm::errs() << "Input vector is empty.\n"; + return nullptr; + } + + try + { + + std::vector *outputTensors = reinterpret_cast*>(this->SerDes->getResponse()); + auto outputs = reinterpret_cast(this->CompiledModel)->run((*reinterpret_cast(this->SerDes->getRequest()))); + for (auto i = outputs.begin(); i != outputs.end(); ++i) + (*(outputTensors)).push_back(*i); + void *rawData = this->SerDes->deserializeUntyped(outputTensors); + return rawData; + } + catch (const c10::Error &e) + { + llvm::errs() << "Error during model evaluation: " << e.what() << "\n"; + return nullptr; + } + } + + template + void PTModelRunner::populateFeatures(const std::pair &var1, + const std::pair &...var2) + { + SerDes->setFeature(var1.first, var1.second); + PTModelRunner::populateFeatures(var2...); + } + +} // namespace MLBridge diff --git a/SerDes/CMakeLists.txt b/SerDes/CMakeLists.txt index 9987d5b..dd11533 100755 --- a/SerDes/CMakeLists.txt +++ b/SerDes/CMakeLists.txt @@ -1,5 +1,7 @@ find_package(Torch REQUIRED) +add_subdirectory(pytorchSerDes) + set(protobuf_MODULE_COMPATIBLE TRUE) find_package(Protobuf CONFIG REQUIRED) @@ -9,18 +11,15 @@ TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp protobufSerDes.cpp -pytorchSerDes.cpp tensorflowSerDes.cpp JSON.cpp ) else() - add_library(SerDesLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp protobufSerDes.cpp tensorflowSerDes.cpp JSON.cpp pytorchSerDes.cpp) - - # add_library(SerDesCLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp JSON.cpp) + add_library(SerDesLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp protobufSerDes.cpp tensorflowSerDes.cpp JSON.cpp ) endif() -target_link_libraries(SerDesLib PRIVATE ${TORCH_LIBRARIES}) -target_compile_options(SerDesLib PRIVATE -fexceptions) +target_link_libraries(SerDesLib PUBLIC PyTorchSerDesLib) + target_include_directories(SerDesLib PUBLIC ${TENSORFLOW_AOT_PATH}/include) target_link_libraries(SerDesLib PRIVATE tf_xla_runtime) diff --git a/SerDes/pytorchSerDes/CMakeLists.txt b/SerDes/pytorchSerDes/CMakeLists.txt new file mode 100755 index 0000000..be48f71 --- /dev/null +++ b/SerDes/pytorchSerDes/CMakeLists.txt @@ -0,0 +1,19 @@ +find_package(Torch REQUIRED) + +# set(protobuf_MODULE_COMPATIBLE TRUE) +# find_package(Protobuf CONFIG REQUIRED) + +if(LLVM_MLBRIDGE) +add_llvm_library(PyTorchSerDesLib +pytorchSerDes.cpp +) + +else() + add_library(PyTorchSerDesLib OBJECT pytorchSerDes.cpp) + + # add_library(SerDesCLib OBJECT TensorSpec.cpp jsonSerDes.cpp bitstreamSerDes.cpp JSON.cpp) +endif() + +target_link_libraries(PyTorchSerDesLib PRIVATE ${TORCH_LIBRARIES}) +target_compile_options(PyTorchSerDesLib PRIVATE -fexceptions) +target_include_directories(PyTorchSerDesLib PUBLIC ${TORCH_INCLUDE_DIRS}) diff --git a/SerDes/pytorchSerDes.cpp b/SerDes/pytorchSerDes/pytorchSerDes.cpp similarity index 55% rename from SerDes/pytorchSerDes.cpp rename to SerDes/pytorchSerDes/pytorchSerDes.cpp index eb98839..5a2e048 100644 --- a/SerDes/pytorchSerDes.cpp +++ b/SerDes/pytorchSerDes/pytorchSerDes.cpp @@ -6,59 +6,72 @@ #include "SerDes/pytorchSerDes.h" #include "SerDes/baseSerDes.h" +#include +#include + +using TensorVec = std::vector; namespace MLBridge { +PytorchSerDes::PytorchSerDes() : BaseSerDes(BaseSerDes::Kind::Pytorch) { + // inputTensors = std::make_shared>(); + // outputTensors = new std::vector(); + + // RequestVoid = std::make_shared>(); + RequestVoid = new TensorVec(); + ResponseVoid = new TensorVec(); +} + void PytorchSerDes::setFeature(const std::string &Name, const int Value) { auto tensor = torch::tensor({Value}, torch::kInt32); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const long Value) { auto tensor = torch::tensor({Value}, torch::kInt64); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const float Value) { auto tensor = torch::tensor({Value}, torch::kFloat32); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const double Value) { auto tensor = torch::tensor({Value}, torch::kFloat64); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::string Value) { std::vector encoded_str(Value.begin(), Value.end()); auto tensor = torch::tensor(encoded_str, torch::kInt8); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const bool Value) { auto tensor = torch::tensor({Value}, torch::kBool); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { auto tensor = torch::tensor(Value, torch::kInt32); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { auto tensor = torch::tensor(Value, torch::kInt64); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { auto tensor = torch::tensor(Value, torch::kFloat32); tensor = tensor.reshape({1, Value.size()}); - inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { auto tensor = torch::tensor(Value, torch::kFloat64); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { @@ -68,13 +81,13 @@ void PytorchSerDes::setFeature(const std::string &Name, const std::vectorinputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } void PytorchSerDes::setFeature(const std::string &Name, const std::vector &Value) { std::vector bool_vec(Value.begin(), Value.end()); auto tensor = torch::tensor(bool_vec, torch::kUInt8); - this->inputTensors->push_back(tensor.clone()); + reinterpret_cast(this->RequestVoid)->push_back(tensor.clone()); } // void PytorchSerDes::setRequest(void *Request) { @@ -82,7 +95,7 @@ void PytorchSerDes::setFeature(const std::string &Name, const std::vector // } void PytorchSerDes::cleanDataStructures() { - this->inputTensors->clear(); // Clear the input vector + reinterpret_cast(this->RequestVoid)->clear(); // Clear the input vector } void *PytorchSerDes::deserializeUntyped(void *Data) { @@ -91,7 +104,7 @@ void *PytorchSerDes::deserializeUntyped(void *Data) { } // Assume Data is a pointer to a vector of tensors - std::vector *serializedTensors = reinterpret_cast *>(Data); + std::vector *serializedTensors = reinterpret_cast(Data); if (serializedTensors->empty()) { return nullptr; @@ -100,22 +113,22 @@ void *PytorchSerDes::deserializeUntyped(void *Data) { auto type_vect = serializedTensors->at(0).dtype(); if (type_vect == torch::kInt32) { - return copyTensorToVect(serializedTensors); + return copyTensorToVect(Data); } else if (type_vect == torch::kInt64) { - return copyTensorToVect(serializedTensors); + return copyTensorToVect(Data); } else if (type_vect == torch::kFloat32) { - return copyTensorToVect(serializedTensors); + return copyTensorToVect(Data); } else if (type_vect == torch::kFloat64) { - return copyTensorToVect(serializedTensors); + return copyTensorToVect(Data); } else if (type_vect == torch::kBool) { - return copyTensorToVect(serializedTensors); + return copyTensorToVect(Data); } else if (type_vect == torch::kInt8) { - return copyTensorToVect(serializedTensors); + return copyTensorToVect(Data); } else { llvm::errs() << "Unsupported tensor dtype.\n"; @@ -124,13 +137,25 @@ void *PytorchSerDes::deserializeUntyped(void *Data) { } void *PytorchSerDes::getSerializedData() { - std::vector serializedData = *(this->outputTensors); + return this->ResponseVoid; // TODO - check + // TensorVec serializedData = *reinterpret_cast(this->ReponseVoid); - // Allocate memory for the output and copy the serialized data - auto *output = new std::vector(serializedData); - return static_cast(output); + // // Allocate memory for the output and copy the serialized data + // auto *output = new TensorVec(serializedData); + // return static_cast(output); } + template + std::vector *PytorchSerDes::copyTensorToVect(void *serializedTensors) { + auto *ret = new std::vector(); + for (const auto &tensor : *reinterpret_cast(serializedTensors)) { + ret->insert(ret->end(), tensor.data_ptr(), tensor.data_ptr() + tensor.numel()); + } + return ret; + } + +void *PytorchSerDes::getRequest() { return this->RequestVoid; } +void *PytorchSerDes::getResponse() { return this->ResponseVoid; } -} // namespace MLBridge +} // namespace MLBridge \ No newline at end of file diff --git a/include/MLModelRunner/PTModelRunner.h b/include/MLModelRunner/PTModelRunner.h index bd9cb93..878c310 100644 --- a/include/MLModelRunner/PTModelRunner.h +++ b/include/MLModelRunner/PTModelRunner.h @@ -3,71 +3,90 @@ #include "MLModelRunner/MLModelRunner.h" #include "SerDes/TensorSpec.h" +// #include "SerDes/baseSerDes.h" #include "SerDes/pytorchSerDes.h" #include "llvm/Support/ErrorHandling.h" -#include -#include // or model_container_runner_cuda.h for CUDA +// #include +// #include // or model_container_runner_cuda.h for CUDA #include #include +namespace MLBridge +{ -namespace MLBridge { + /// PTModelRunner - PyTorch Compiled model implementation of the + /// MLModelRunner. It uses an AOT-compiled model for efficient execution. + class PTModelRunner final : public MLModelRunner + { + public: + // New constructor that takes the model path as an input + PTModelRunner(const std::string &modelPath, llvm::LLVMContext &Ctx); + // { + // this->SerDes = new PytorchSerDes(); -/// PTModelRunner - PyTorch Compiled model implementation of the -/// MLModelRunner. It uses an AOT-compiled model for efficient execution. -class PTModelRunner final : public MLModelRunner { -public: - // New constructor that takes the model path as an input - PTModelRunner(const std::string &modelPath, llvm::LLVMContext &Ctx) - : MLModelRunner(MLModelRunner::Kind::PTAOT, SerDesKind::Pytorch, &Ctx) - { - this->SerDes = new PytorchSerDes(modelPath); - } + // c10::InferenceMode mode; + // this->CompiledModel = new torch::inductor::AOTIModelContainerRunnerCpu(modelPath); + // } - virtual ~PTModelRunner() = default; + virtual ~PTModelRunner() = default; - virtual void requestExit() override { - llvm_unreachable("requestExit() is not supported in PTModelRunner"); - } - - static bool classof(const MLModelRunner *R) { - return R->getKind() == MLModelRunner::Kind::PTAOT; - } - - - - template - void populateFeatures(const std::pair &var1, - const std::pair &...var2); - - void populateFeatures() {} - - - void *evaluateUntyped() override; - PytorchSerDes *SerDes; -}; + virtual void requestExit() override + { + llvm_unreachable("requestExit() is not supported in PTModelRunner"); + } + static bool classof(const MLModelRunner *R) + { + return R->getKind() == MLModelRunner::Kind::PTAOT; + } -void* PTModelRunner::evaluateUntyped() { -auto outputs = SerDes->CompiledModel->run((*(this->SerDes->inputTensors))); -for (auto i = outputs.begin(); i != outputs.end(); ++i) - (*(this->SerDes->outputTensors)).push_back(*i); -void* rawData = SerDes->deserializeUntyped(SerDes->outputTensors); -return rawData; + template + void populateFeatures(const std::pair &var1, + const std::pair &...var2); + + void populateFeatures() {} + + void *evaluateUntyped() override; + PytorchSerDes *SerDes; + // Compiled model container added to the PTModelRunner + private: + void *CompiledModel; + // torch::inductor::AOTIModelContainerRunnerCpu *CompiledModel; + }; + + // void *PTModelRunner::evaluateUntyped() + // { + + // if ((*(this->SerDes->inputTensors)).empty()) + // { + // llvm::errs() << "Input vector is empty.\n"; + // return nullptr; + // } + + // try + // { + // auto outputs = this->CompiledModel->run((*(this->SerDes->inputTensors))); + // for (auto i = outputs.begin(); i != outputs.end(); ++i) + // (*(this->SerDes->outputTensors)).push_back(*i); + // void *rawData = this->SerDes->deserializeUntyped(this->SerDes->outputTensors); + // return rawData; + // } + // catch (const c10::Error &e) + // { + // llvm::errs() << "Error during model evaluation: " << e.what() << "\n"; + // return nullptr; + // } + // } + + // template + // void PTModelRunner::populateFeatures(const std::pair &var1, + // const std::pair &...var2) + // { + // SerDes->setFeature(var1.first, var1.second); + // PTModelRunner::populateFeatures(var2...); + // } - } catch (const c10::Error& e) { - llvm::errs() << "Error during model evaluation: " << e.what() << "\n"; - return nullptr; - } -} - -template - void PTModelRunner::populateFeatures(const std::pair &var1, - const std::pair &...var2) { - SerDes->setFeature(var1.first, var1.second); - PTModelRunner::populateFeatures(var2...); - } } // namespace MLBridge -#endif // TFMODELRUNNER_H +#endif // TFMODELRUNNER_H \ No newline at end of file diff --git a/include/SerDes/pytorchSerDes.h b/include/SerDes/pytorchSerDes.h index 77667f7..a67b148 100644 --- a/include/SerDes/pytorchSerDes.h +++ b/include/SerDes/pytorchSerDes.h @@ -6,23 +6,20 @@ #include // #include -#include -#include +// #include +// #include namespace MLBridge { class PytorchSerDes : public BaseSerDes { public: - PytorchSerDes() : BaseSerDes(BaseSerDes::Kind::Pytorch) {} - // Constructor that takes the model path and loads the model - PytorchSerDes(const std::string &modelPath) : BaseSerDes(BaseSerDes::Kind::Pytorch) { - c10::InferenceMode mode; - this->CompiledModel = new torch::inductor::AOTIModelContainerRunnerCpu(modelPath); - inputTensors = std::make_shared>(); - outputTensors = new std::vector(); - } + PytorchSerDes(); + // { + // inputTensors = std::make_shared>(); + // outputTensors = new std::vector(); + // } - ~PytorchSerDes() { delete CompiledModel; } + ~PytorchSerDes() {} static bool classof(const BaseSerDes *S) { return S->getKind() == BaseSerDes::Kind::Pytorch; @@ -34,28 +31,31 @@ class PytorchSerDes : public BaseSerDes { SUPPORTED_TYPES(SET_FEATURE) #undef SET_FEATURE - void setRequest(void *request) override { - CompiledModel = - reinterpret_cast(request); + // NEW + void *getRequest() override; + void *getResponse() override; + + void setRequest(void *request){ } template - std::vector *copyTensorToVect(std::vector *serializedTensors) { - auto *ret = new std::vector(); - for (const auto &tensor : *serializedTensors) { - ret->insert(ret->end(), tensor.data_ptr(), tensor.data_ptr() + tensor.numel()); - } - return ret; - } - + std::vector *copyTensorToVect(void *serializedTensors); + // { + // auto *ret = new std::vector(); + // for (const auto &tensor : *serializedTensors) { + // ret->insert(ret->end(), tensor.data_ptr(), tensor.data_ptr() + tensor.numel()); + // } + // return ret; + // } + void *getSerializedData() override; void cleanDataStructures() override; void *deserializeUntyped(void *data) override; - std::vector *outputTensors; // Storage for the PyTorch output tensor - std::shared_ptr> inputTensors; // Declaration of the input tensor - torch::inductor::AOTIModelContainerRunnerCpu* CompiledModel; + // std::vector *outputTensors; // Storage for the PyTorch output tensor + // std::shared_ptr> inputTensors; // Declaration of the input tensor + }; } // namespace MLBridge