From 9bc597230fea2d78050fb774b5793f40f7fe105c Mon Sep 17 00:00:00 2001 From: "Skrebkov, Artemy" Date: Wed, 27 Nov 2024 17:08:10 +0000 Subject: [PATCH 01/11] Parse dynamic dims from metadata correctly Signed-off-by: Skrebkov, Artemy --- .../src/compiler_adapter/src/ze_graph_ext_wrappers.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp b/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp index f6366a2509747b..952edb4e7c6929 100644 --- a/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp +++ b/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp @@ -410,8 +410,13 @@ static IODescriptor getIODescriptor(const ze_graph_argument_properties_3_t& arg, shapeFromCompiler.push_back(arg.dims[id]); } if (metadata.has_value()) { + const auto dynamicDim = std::numeric_limits::max(); for (uint32_t id = 0; id < metadata->shape_size; id++) { - shapeFromIRModel.push_back(metadata->shape[id]); + if (metadata->shape[id] != dynamicDim) { + shapeFromIRModel.push_back(metadata->shape[id]); + } else { + shapeFromIRModel.push_back(-1); + } } } @@ -433,7 +438,7 @@ static IODescriptor getIODescriptor(const ze_graph_argument_properties_3_t& arg, return {std::move(nameFromCompiler), precision, - std::move(shapeFromCompiler), + shapeFromCompiler, isStateInput, isStateOutput, isShapeTensor, From 16360e061234500ae2488fc11daf62bbf5baf994 Mon Sep 17 00:00:00 2001 From: "Skrebkov, Artemy" Date: Wed, 27 Nov 2024 17:42:45 +0000 Subject: [PATCH 02/11] Enable ignoring square brackets when parsing data_shape Signed-off-by: Skrebkov, Artemy --- .../tools/single-image-test/main.cpp | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/plugins/intel_npu/tools/single-image-test/main.cpp b/src/plugins/intel_npu/tools/single-image-test/main.cpp index 2454e9afd40e0d..bc840e50d5b503 100644 --- a/src/plugins/intel_npu/tools/single-image-test/main.cpp +++ b/src/plugins/intel_npu/tools/single-image-test/main.cpp @@ -1550,8 +1550,8 @@ std::pair runInfer(ov::InferRequest& inferRequest, ov::Compi TensorMap out; for (const auto& outputInfo : compiledModel.outputs()) { - const std::string layer_name = outputInfo.get_any_name(); - out.insert({layer_name, inferRequest.get_tensor(layer_name)}); + const std::string layerName = outputInfo.get_any_name(); + out.insert({layerName, inferRequest.get_tensor(layerName)}); } ProfVec profData{}; @@ -1788,11 +1788,17 @@ bool testMeanIoU(const TensorMap& outputs, const TensorMap& references, const La } static ov::Shape parseDataShape(const std::string& dataShapeStr) { - std::vector dataShape; - std::istringstream ss(dataShapeStr); - std::string token; - while (std::getline(ss, token, ',')) { - dataShape.push_back(std::stoul(token)); + std::vector dataShape; + std::stringstream ss(dataShapeStr); + + char ch; // To discard non-numeric characters + int64_t dim; + while (ss >> ch) { + if (std::isdigit(ch)) { + ss.putback(ch); + ss >> dim; + dataShape.push_back(dim); + } } return ov::Shape(dataShape); } @@ -1887,11 +1893,11 @@ static int runSingleImageTest() { auto model = core.read_model(FLAGS_network); nameIOTensors(model); - auto inputs_info = std::const_pointer_cast(model)->inputs(); - InputsInfo info_map; + auto inputsInfo = std::const_pointer_cast(model)->inputs(); + InputsInfo infoMap; std::cout << "Performing reshape" << std::endl; - reshape(std::move(inputs_info), info_map, model, FLAGS_shape, + reshape(std::move(inputsInfo), infoMap, model, FLAGS_shape, FLAGS_override_model_batch_size, FLAGS_device); ov::preprocess::PrePostProcessor ppp(model); From a1369c4b2166e84205e51980d4f9b7877af53026 Mon Sep 17 00:00:00 2001 From: "Skrebkov, Artemy" Date: Thu, 28 Nov 2024 17:56:16 +0000 Subject: [PATCH 03/11] Fix check for dynamic dim Signed-off-by: Skrebkov, Artemy --- src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp index a0e5d2d11c1fef..254aa21ed1c3da 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp @@ -54,9 +54,10 @@ void check_level_zero_attributes_match(const IODescriptor& ioDescriptor, const A "Maximum number of dimensions supported: " + std::to_string(ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE) + '\n' + "Given: " + std::to_string(ovDimensions.size())); + const uint32_t dynamicDim = std::numeric_limits::max(); for (size_t index = 0; index < ovDimensions.size(); ++index) { OPENVINO_ASSERT( - ioDescriptor.shapeFromCompiler.is_dynamic() || ovDimensions[index] == zeDescriptor.info.dims[index], + ovDimensions[index] == zeDescriptor.info.dims[index] || zeDescriptor.info.dims[index] == dynamicDim, "Shape mismatch for input/output named " + ioDescriptor.nameFromCompiler); } for (size_t index = ovDimensions.size(); index < ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE; ++index) { From c02784289d6a8ab37331beab25b75281e9dc3c50 Mon Sep 17 00:00:00 2001 From: "Skrebkov, Artemy" Date: Fri, 20 Dec 2024 14:56:05 +0000 Subject: [PATCH 04/11] Set bounded dims for shapeFromIRModel Signed-off-by: Skrebkov, Artemy --- .../compiler_adapter/src/ze_graph_ext_wrappers.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp b/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp index 952edb4e7c6929..9f1619515fd19b 100644 --- a/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp +++ b/src/plugins/intel_npu/src/compiler_adapter/src/ze_graph_ext_wrappers.cpp @@ -12,7 +12,9 @@ #include "intel_npu/utils/zero/zero_api.hpp" #include "intel_npu/utils/zero/zero_result.hpp" #include "intel_npu/utils/zero/zero_wrappers.hpp" +#include "openvino/core/dimension.hpp" #include "openvino/core/model.hpp" +#include "openvino/core/partial_shape.hpp" #define NotSupportQuery(T) (T <= ZE_GRAPH_EXT_VERSION_1_2) @@ -400,7 +402,8 @@ ze_graph_handle_t ZeGraphExtWrappers::getGraphHandle(const std::vector& static IODescriptor getIODescriptor(const ze_graph_argument_properties_3_t& arg, const std::optional& metadata) { ov::element::Type_t precision = toOVElementType(arg.devicePrecision); - ov::Shape shapeFromCompiler, shapeFromIRModel; + ov::Shape shapeFromCompiler; + ov::PartialShape shapeFromIRModel; std::unordered_set outputTensorNames; for (uint32_t id = 0; id < arg.associated_tensor_names_count; id++) { @@ -411,11 +414,15 @@ static IODescriptor getIODescriptor(const ze_graph_argument_properties_3_t& arg, } if (metadata.has_value()) { const auto dynamicDim = std::numeric_limits::max(); + shapeFromIRModel.reserve(metadata->shape_size); for (uint32_t id = 0; id < metadata->shape_size; id++) { if (metadata->shape[id] != dynamicDim) { shapeFromIRModel.push_back(metadata->shape[id]); } else { - shapeFromIRModel.push_back(-1); + // lower bound is ignored, so we set it to 1 just to satisfy the Dimension constructor, + // upper bound is set to the value from shapeFromCompiler as it is filled with upper bounds + // in case of dynamic dimensions + shapeFromIRModel.push_back(ov::Dimension(1, shapeFromCompiler[id])); } } } From 5ebcc5842f4a9216e0fe03ea1a97a2b2f94c3345 Mon Sep 17 00:00:00 2001 From: "Skrebkov, Artemy" Date: Sun, 22 Dec 2024 22:53:57 +0000 Subject: [PATCH 05/11] zeDescriptor.dims cannot be dynamic Signed-off-by: Skrebkov, Artemy --- src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp index 254aa21ed1c3da..a1fd77c514c174 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp @@ -54,10 +54,9 @@ void check_level_zero_attributes_match(const IODescriptor& ioDescriptor, const A "Maximum number of dimensions supported: " + std::to_string(ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE) + '\n' + "Given: " + std::to_string(ovDimensions.size())); - const uint32_t dynamicDim = std::numeric_limits::max(); for (size_t index = 0; index < ovDimensions.size(); ++index) { OPENVINO_ASSERT( - ovDimensions[index] == zeDescriptor.info.dims[index] || zeDescriptor.info.dims[index] == dynamicDim, + ovDimensions[index] == zeDescriptor.info.dims[index], "Shape mismatch for input/output named " + ioDescriptor.nameFromCompiler); } for (size_t index = ovDimensions.size(); index < ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE; ++index) { From 2ce1454ff29286a86e3a86d57832fa3d01033411 Mon Sep 17 00:00:00 2001 From: "Skrebkov, Artemy" Date: Mon, 23 Dec 2024 22:34:35 +0000 Subject: [PATCH 06/11] Fix codestyle check Signed-off-by: Skrebkov, Artemy --- src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp index a1fd77c514c174..f488ff8553c4d4 100644 --- a/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp +++ b/src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp @@ -55,9 +55,8 @@ void check_level_zero_attributes_match(const IODescriptor& ioDescriptor, const A '\n' + "Given: " + std::to_string(ovDimensions.size())); for (size_t index = 0; index < ovDimensions.size(); ++index) { - OPENVINO_ASSERT( - ovDimensions[index] == zeDescriptor.info.dims[index], - "Shape mismatch for input/output named " + ioDescriptor.nameFromCompiler); + OPENVINO_ASSERT(ovDimensions[index] == zeDescriptor.info.dims[index], + "Shape mismatch for input/output named " + ioDescriptor.nameFromCompiler); } for (size_t index = ovDimensions.size(); index < ZE_MAX_GRAPH_ARGUMENT_DIMENSIONS_SIZE; ++index) { OPENVINO_ASSERT(zeDescriptor.info.dims[index] == 0 || zeDescriptor.info.dims[index] == 1, From 13c505bb1e55e3dbf4a1a28784742e48a08c6e03 Mon Sep 17 00:00:00 2001 From: "Skrebkov, Artemy" Date: Thu, 16 Jan 2025 22:22:55 +0000 Subject: [PATCH 07/11] Set isDynamicNetwork for compiled networks Signed-off-by: Skrebkov, Artemy --- samples/cpp/benchmark_app/main.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index 2b51b6f1f87251..da2877ce7bda11 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -523,9 +523,7 @@ int main(int argc, char* argv[]) { } } auto result = std::find_if(config.begin(), config.end(), [&](const std::pair& item) { - if (device_name.find(item.first) == 0) - return true; - return false; + return device_name.find(item.first) == 0; }); ov::AnyMap device_config = {}; if (result != config.end()) @@ -548,6 +546,13 @@ int main(int argc, char* argv[]) { } bool isDynamicNetwork = false; + auto areNetworkInputsDynamic = [](benchmark_app::InputsInfo& input_info) { + return std::any_of(input_info.begin(), + input_info.end(), + [](const std::pair& i) { + return i.second.partialShape.is_dynamic(); + }); + }; if (FLAGS_load_from_file && !isNetworkCompiled) { if (!FLAGS_mean_values.empty() || !FLAGS_scale_values.empty()) { @@ -722,12 +727,7 @@ int main(int argc, char* argv[]) { model = preproc.build(); // Check if network has dynamic shapes - auto input_info = app_inputs_info[0]; - isDynamicNetwork = std::any_of(input_info.begin(), - input_info.end(), - [](const std::pair& i) { - return i.second.partialShape.is_dynamic(); - }); + isDynamicNetwork = areNetworkInputsDynamic(app_inputs_info.at(0)); topology_name = model->get_friendly_name(); @@ -789,6 +789,7 @@ int main(int argc, char* argv[]) { FLAGS_scale_values, FLAGS_mean_values, compiledModel.inputs()); + isDynamicNetwork = areNetworkInputsDynamic(app_inputs_info.at(0)); batchSize = get_batch_size(app_inputs_info.at(0)); warn_if_no_batch(app_inputs_info.at(0)); From f047dd15680ea7217105abe01dcb15dcaa5063ea Mon Sep 17 00:00:00 2001 From: Artemy Skrebkov Date: Fri, 17 Jan 2025 12:11:06 +0000 Subject: [PATCH 08/11] Update samples/cpp/benchmark_app/main.cpp Co-authored-by: Pawel Raasz --- samples/cpp/benchmark_app/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index da2877ce7bda11..a43eed30d5b2a3 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -546,7 +546,7 @@ int main(int argc, char* argv[]) { } bool isDynamicNetwork = false; - auto areNetworkInputsDynamic = [](benchmark_app::InputsInfo& input_info) { + auto areNetworkInputsDynamic = [](const benchmark_app::InputsInfo& input_info) { return std::any_of(input_info.begin(), input_info.end(), [](const std::pair& i) { From 15fcd8d0980d4d16e873f8ed421fadd16c69ab23 Mon Sep 17 00:00:00 2001 From: Artemy Skrebkov Date: Fri, 17 Jan 2025 12:11:14 +0000 Subject: [PATCH 09/11] Update samples/cpp/benchmark_app/main.cpp Co-authored-by: Pawel Raasz --- samples/cpp/benchmark_app/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index a43eed30d5b2a3..25bdb3feaeb2de 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -549,7 +549,7 @@ int main(int argc, char* argv[]) { auto areNetworkInputsDynamic = [](const benchmark_app::InputsInfo& input_info) { return std::any_of(input_info.begin(), input_info.end(), - [](const std::pair& i) { + [](const auto& info) { return i.second.partialShape.is_dynamic(); }); }; From 1e7a3dbc7205ea81d4bee700c7230707f4e58fbe Mon Sep 17 00:00:00 2001 From: "Skrebkov, Artemy" Date: Fri, 17 Jan 2025 12:13:23 +0000 Subject: [PATCH 10/11] Fix compilation after applying review comments Signed-off-by: Skrebkov, Artemy --- samples/cpp/benchmark_app/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index 25bdb3feaeb2de..141b6042dc2fb4 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -550,7 +550,7 @@ int main(int argc, char* argv[]) { return std::any_of(input_info.begin(), input_info.end(), [](const auto& info) { - return i.second.partialShape.is_dynamic(); + return info.second.partialShape.is_dynamic(); }); }; From 9a46d79d035588f548e3db701fe3178264ac20d5 Mon Sep 17 00:00:00 2001 From: "Skrebkov, Artemy" Date: Fri, 17 Jan 2025 12:20:49 +0000 Subject: [PATCH 11/11] Fix clang format Signed-off-by: Skrebkov, Artemy --- samples/cpp/benchmark_app/main.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp index 141b6042dc2fb4..4050f54f867969 100644 --- a/samples/cpp/benchmark_app/main.cpp +++ b/samples/cpp/benchmark_app/main.cpp @@ -547,11 +547,9 @@ int main(int argc, char* argv[]) { bool isDynamicNetwork = false; auto areNetworkInputsDynamic = [](const benchmark_app::InputsInfo& input_info) { - return std::any_of(input_info.begin(), - input_info.end(), - [](const auto& info) { - return info.second.partialShape.is_dynamic(); - }); + return std::any_of(input_info.begin(), input_info.end(), [](const auto& info) { + return info.second.partialShape.is_dynamic(); + }); }; if (FLAGS_load_from_file && !isNetworkCompiled) {