From 41ecf3190ec0e337e36c5d23873107ff41492634 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Thu, 6 Mar 2025 01:37:33 -0800 Subject: [PATCH 1/3] Add support for parsing AUTO, HETERO and MULTI from json config --- .../providers/openvino/backends/basic_backend.cc | 15 +++++++++++++++ .../openvino/openvino_provider_factory.cc | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 14a4a466613dd..7c83fac795bef 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -222,6 +222,15 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { } } } + auto find_device_type_mode = [&](const std::string& device_type) -> std::string { + std::string device_mode=""; + auto delimiter_pos = device_type.find(':'); + if (delimiter_pos != std::string::npos) { + std::stringstream str_stream(device_type.substr(0, delimiter_pos)); + std::getline(str_stream, device_mode, ','); + } + return device_mode; + }; // Parse device types like "AUTO:CPU,GPU" and extract individual devices auto parse_individual_devices = [&](const std::string& device_type) -> std::vector { @@ -270,8 +279,12 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { if (session_context_.device_type.find("AUTO") == 0 || session_context_.device_type.find("HETERO") == 0 || session_context_.device_type.find("MULTI") == 0) { + //// Parse to get the device mode (e.g., "AUTO:CPU,GPU" -> "AUTO") + auto device_mode = find_device_type_mode(session_context_.device_type); // Parse individual devices (e.g., "AUTO:CPU,GPU" -> ["CPU", "GPU"]) auto individual_devices = parse_individual_devices(session_context_.device_type); + if(!device_mode.empty()) individual_devices.emplace_back(device_mode); + // Set properties only for individual devices (e.g., "CPU", "GPU") for (const std::string& device : individual_devices) { if (target_config.count(device)) { @@ -282,6 +295,8 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { } } } else { + std::unordered_set valid_ov_devices = {"CPU", "GPU", "NPU", "AUTO", "HETERO", "MULTI"}; + if (target_config.count(session_context_.device_type)) { auto supported_properties = OVCore::Get()->core.get_property(session_context_.device_type, ov::supported_properties); diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 5809ca2cf1c7b..8799cf543ab0f 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -216,9 +216,9 @@ struct OpenVINO_Provider : Provider { for (auto& [key, value] : json_config.items()) { ov::AnyMap inner_map; - + std::unordered_set valid_ov_devices = {"CPU", "GPU", "NPU", "AUTO", "HETERO", "MULTI"}; // Ensure the key is one of "CPU", "GPU", or "NPU" - if (key != "CPU" && key != "GPU" && key != "NPU") { + if (valid_ov_devices.find(key) == valid_ov_devices.end()) { LOGS_DEFAULT(WARNING) << "Unsupported device key: " << key << ". Skipping entry.\n"; continue; } From 1c950b1d30caeb0050f7a5526ccb289e3b61fa13 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Sun, 9 Mar 2025 22:21:38 -0700 Subject: [PATCH 2/3] Fix lint issues --- .../core/providers/openvino/backends/basic_backend.cc | 6 ++++-- .../core/providers/openvino/openvino_provider_factory.cc | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 7c83fac795bef..cd6f0be11f28c 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -2,6 +2,8 @@ // Licensed under the MIT License #include +#include + #include #include #include @@ -223,7 +225,7 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { } } auto find_device_type_mode = [&](const std::string& device_type) -> std::string { - std::string device_mode=""; + std::string device_mode = ""; auto delimiter_pos = device_type.find(':'); if (delimiter_pos != std::string::npos) { std::stringstream str_stream(device_type.substr(0, delimiter_pos)); @@ -283,7 +285,7 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { auto device_mode = find_device_type_mode(session_context_.device_type); // Parse individual devices (e.g., "AUTO:CPU,GPU" -> ["CPU", "GPU"]) auto individual_devices = parse_individual_devices(session_context_.device_type); - if(!device_mode.empty()) individual_devices.emplace_back(device_mode); + if (!device_mode.empty()) individual_devices.emplace_back(device_mode); // Set properties only for individual devices (e.g., "CPU", "GPU") for (const std::string& device : individual_devices) { diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 8799cf543ab0f..b8f9aec3a68ed 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -2,6 +2,8 @@ // Licensed under the MIT License #include +#include + #include #include "core/providers/shared_library/provider_api.h" #include "core/providers/openvino/openvino_provider_factory.h" @@ -216,7 +218,7 @@ struct OpenVINO_Provider : Provider { for (auto& [key, value] : json_config.items()) { ov::AnyMap inner_map; - std::unordered_set valid_ov_devices = {"CPU", "GPU", "NPU", "AUTO", "HETERO", "MULTI"}; + std::set valid_ov_devices = {"CPU", "GPU", "NPU", "AUTO", "HETERO", "MULTI"}; // Ensure the key is one of "CPU", "GPU", or "NPU" if (valid_ov_devices.find(key) == valid_ov_devices.end()) { LOGS_DEFAULT(WARNING) << "Unsupported device key: " << key << ". Skipping entry.\n"; From 2d4d5000f0c34fb751a7485ae6caa4434bda4879 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Sun, 16 Mar 2025 13:28:18 -0700 Subject: [PATCH 3/3] Address review comments --- onnxruntime/core/providers/openvino/backends/basic_backend.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index cd6f0be11f28c..18182284181e5 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -282,7 +282,9 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { session_context_.device_type.find("HETERO") == 0 || session_context_.device_type.find("MULTI") == 0) { //// Parse to get the device mode (e.g., "AUTO:CPU,GPU" -> "AUTO") + std::unordered_set supported_mode = {"AUTO", "HETERO", "MULTI"}; auto device_mode = find_device_type_mode(session_context_.device_type); + ORT_ENFORCE(supported_mode.find(device_mode)!=supported_mode.end(), " Invalid device mode is passed : " , session_context_.device_type); // Parse individual devices (e.g., "AUTO:CPU,GPU" -> ["CPU", "GPU"]) auto individual_devices = parse_individual_devices(session_context_.device_type); if (!device_mode.empty()) individual_devices.emplace_back(device_mode); @@ -297,8 +299,6 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { } } } else { - std::unordered_set valid_ov_devices = {"CPU", "GPU", "NPU", "AUTO", "HETERO", "MULTI"}; - if (target_config.count(session_context_.device_type)) { auto supported_properties = OVCore::Get()->core.get_property(session_context_.device_type, ov::supported_properties);