diff --git a/src/cpp/include/openvino/genai/text2image/unet2d_condition_model.hpp b/src/cpp/include/openvino/genai/text2image/unet2d_condition_model.hpp index b3cfe1d364..9ac6cbba83 100644 --- a/src/cpp/include/openvino/genai/text2image/unet2d_condition_model.hpp +++ b/src/cpp/include/openvino/genai/text2image/unet2d_condition_model.hpp @@ -7,13 +7,13 @@ #include #include -#include "openvino/genai/visibility.hpp" - #include "openvino/core/any.hpp" #include "openvino/core/model.hpp" #include "openvino/runtime/tensor.hpp" #include "openvino/runtime/infer_request.hpp" #include "openvino/runtime/properties.hpp" + +#include "openvino/genai/visibility.hpp" #include "openvino/genai/lora_adapter.hpp" namespace ov { diff --git a/src/cpp/src/lora_adapter.cpp b/src/cpp/src/lora_adapter.cpp index e66800e1c9..a2a2f56622 100644 --- a/src/cpp/src/lora_adapter.cpp +++ b/src/cpp/src/lora_adapter.cpp @@ -27,10 +27,11 @@ #include "openvino/pass/pattern/op/wrap_type.hpp" #include "openvino/pass/graph_rewrite.hpp" #include "openvino/pass/manager.hpp" -#include "openvino/runtime/core.hpp" #include "openvino/genai/lora_adapter.hpp" +#include "utils.hpp" + extern "C" { #include "safetensors.h" } @@ -561,6 +562,7 @@ class InferRequestSignatureCache { } void insert (const Signature& signature, std::shared_ptr model) { + ov::Core core = ov::genai::utils::singleton_core(); requests[signature] = core.compile_model(model, device).create_infer_request(); } @@ -589,7 +591,6 @@ class InferRequestSignatureCache { private: - ov::Core core; std::unordered_map requests; std::string device; }; diff --git a/src/cpp/src/text2image/models/autoencoder_kl.cpp b/src/cpp/src/text2image/models/autoencoder_kl.cpp index 018d996c6c..080bbd7931 100644 --- a/src/cpp/src/text2image/models/autoencoder_kl.cpp +++ b/src/cpp/src/text2image/models/autoencoder_kl.cpp @@ -13,8 +13,11 @@ #include "openvino/op/multiply.hpp" #include "openvino/op/constant.hpp" +#include "utils.hpp" + #include "json_utils.hpp" #include "lora_helper.hpp" + namespace ov { namespace genai { @@ -34,7 +37,8 @@ AutoencoderKL::Config::Config(const std::string& config_path) { AutoencoderKL::AutoencoderKL(const std::string& root_dir) : m_config(root_dir + "/config.json") { - m_model = ov::Core().read_model(root_dir + "/openvino_model.xml"); + ov::Core core = utils::singleton_core(); + m_model = core.read_model(root_dir + "/openvino_model.xml"); // apply VaeImageProcessor postprocessing steps by merging them into the VAE decoder model merge_vae_image_processor(); } @@ -69,7 +73,8 @@ AutoencoderKL& AutoencoderKL::reshape(int batch_size, int height, int width) { AutoencoderKL& AutoencoderKL::compile(const std::string& device, const ov::AnyMap& properties) { OPENVINO_ASSERT(m_model, "Model has been already compiled. Cannot re-compile already compiled model"); - ov::CompiledModel compiled_model = ov::Core().compile_model(m_model, device, properties); + ov::Core core = utils::singleton_core(); + ov::CompiledModel compiled_model = core.compile_model(m_model, device, properties); m_request = compiled_model.create_infer_request(); // release the original model m_model.reset(); diff --git a/src/cpp/src/text2image/models/clip_text_model.cpp b/src/cpp/src/text2image/models/clip_text_model.cpp index 50af860035..65043a1ac4 100644 --- a/src/cpp/src/text2image/models/clip_text_model.cpp +++ b/src/cpp/src/text2image/models/clip_text_model.cpp @@ -5,10 +5,9 @@ #include -#include "openvino/runtime/core.hpp" - #include "json_utils.hpp" #include "lora_helper.hpp" +#include "utils.hpp" namespace ov { namespace genai { @@ -28,7 +27,8 @@ CLIPTextModel::Config::Config(const std::string& config_path) { CLIPTextModel::CLIPTextModel(const std::string root_dir) : m_clip_tokenizer(root_dir + "/../tokenizer"), m_config(root_dir + "/config.json") { - m_model = ov::Core().read_model(root_dir + "/openvino_model.xml"); + ov::Core core = utils::singleton_core(); + m_model = core.read_model(root_dir + "/openvino_model.xml"); } CLIPTextModel::CLIPTextModel(const std::string& root_dir, @@ -64,7 +64,8 @@ CLIPTextModel& CLIPTextModel::reshape(int batch_size) { CLIPTextModel& CLIPTextModel::compile(const std::string& device, const ov::AnyMap& properties) { OPENVINO_ASSERT(m_model, "Model has been already compiled. Cannot re-compile already compiled model"); - ov::CompiledModel compiled_model = ov::Core().compile_model(m_model, device, properties); + ov::Core core = utils::singleton_core(); + ov::CompiledModel compiled_model = core.compile_model(m_model, device, properties); m_request = compiled_model.create_infer_request(); // release the original model m_model.reset(); diff --git a/src/cpp/src/text2image/models/clip_text_model_with_projection.cpp b/src/cpp/src/text2image/models/clip_text_model_with_projection.cpp index 2dabca9e7d..61418d8e54 100644 --- a/src/cpp/src/text2image/models/clip_text_model_with_projection.cpp +++ b/src/cpp/src/text2image/models/clip_text_model_with_projection.cpp @@ -5,9 +5,8 @@ #include -#include "openvino/runtime/core.hpp" - #include "json_utils.hpp" +#include "utils.hpp" namespace ov { namespace genai { @@ -27,7 +26,8 @@ CLIPTextModelWithProjection::Config::Config(const std::string& config_path) { CLIPTextModelWithProjection::CLIPTextModelWithProjection(const std::string root_dir) : m_clip_tokenizer(root_dir + "/../tokenizer_2"), m_config(root_dir + "/config.json") { - m_model = ov::Core().read_model(root_dir + "/openvino_model.xml"); + ov::Core core = utils::singleton_core(); + m_model = core.read_model(root_dir + "/openvino_model.xml"); } CLIPTextModelWithProjection::CLIPTextModelWithProjection(const std::string& root_dir, @@ -57,7 +57,8 @@ CLIPTextModelWithProjection& CLIPTextModelWithProjection::reshape(int batch_size CLIPTextModelWithProjection& CLIPTextModelWithProjection::compile(const std::string& device, const ov::AnyMap& properties) { OPENVINO_ASSERT(m_model, "Model has been already compiled. Cannot re-compile already compiled model"); - ov::CompiledModel compiled_model = ov::Core().compile_model(m_model, device, properties); + ov::Core core = utils::singleton_core(); + ov::CompiledModel compiled_model = core.compile_model(m_model, device, properties); m_request = compiled_model.create_infer_request(); // release the original model m_model.reset(); diff --git a/src/cpp/src/text2image/models/unet2d_condition_model.cpp b/src/cpp/src/text2image/models/unet2d_condition_model.cpp index d84036e35c..d9053cf62b 100644 --- a/src/cpp/src/text2image/models/unet2d_condition_model.cpp +++ b/src/cpp/src/text2image/models/unet2d_condition_model.cpp @@ -5,10 +5,9 @@ #include -#include "openvino/runtime/core.hpp" - #include "json_utils.hpp" #include "lora_helper.hpp" +#include "utils.hpp" namespace ov { namespace genai { @@ -28,7 +27,8 @@ UNet2DConditionModel::Config::Config(const std::string& config_path) { UNet2DConditionModel::UNet2DConditionModel(const std::string root_dir) : m_config(root_dir + "/config.json") { - m_model = ov::Core().read_model(root_dir + "/openvino_model.xml"); + ov::Core core = utils::singleton_core(); + m_model = core.read_model(root_dir + "/openvino_model.xml"); // compute VAE scale factor m_vae_scale_factor = std::pow(2, m_config.block_out_channels.size() - 1); } @@ -86,7 +86,8 @@ UNet2DConditionModel& UNet2DConditionModel::reshape(int batch_size, int height, UNet2DConditionModel& UNet2DConditionModel::compile(const std::string& device, const ov::AnyMap& properties) { OPENVINO_ASSERT(m_model, "Model has been already compiled. Cannot re-compile already compiled model"); - ov::CompiledModel compiled_model = ov::Core().compile_model(m_model, device, properties); + ov::Core core = utils::singleton_core(); + ov::CompiledModel compiled_model = core.compile_model(m_model, device, properties); m_request = compiled_model.create_infer_request(); // release the original model m_model.reset(); diff --git a/src/cpp/src/utils.cpp b/src/cpp/src/utils.cpp index e7f58a015e..ab5853403f 100644 --- a/src/cpp/src/utils.cpp +++ b/src/cpp/src/utils.cpp @@ -259,6 +259,12 @@ void slice_matmul_statefull_model(std::shared_ptr model) { matmul->input(0).replace_source_output(slice); } } + +ov::Core singleton_core() { + static ov::Core core; + return core; +} + } // namespace utils } // namespace genai } // namespace ov diff --git a/src/cpp/src/utils.hpp b/src/cpp/src/utils.hpp index de7296ac00..5279520a29 100644 --- a/src/cpp/src/utils.hpp +++ b/src/cpp/src/utils.hpp @@ -4,6 +4,8 @@ #pragma once #include "openvino/genai/llm_pipeline.hpp" +#include "openvino/runtime/core.hpp" + #include "visual_language/processor_config.hpp" namespace ov { @@ -57,6 +59,9 @@ std::pair split_core_complile_config(const ov::AnyMap& p ov::genai::TokenizedInputs subtract_chat_tokenized_inputs(const ov::genai::TokenizedInputs& minuend, const ov::genai::TokenizedInputs& subtrahend); void slice_matmul_statefull_model(std::shared_ptr model); + +ov::Core singleton_core(); + } // namespace utils } // namespace genai } // namespace ov