diff --git a/backends/openvino/CMakeLists.txt b/backends/openvino/CMakeLists.txt index 3f2199b634d..017da3d92fe 100644 --- a/backends/openvino/CMakeLists.txt +++ b/backends/openvino/CMakeLists.txt @@ -69,4 +69,3 @@ target_link_options(openvino_backend PRIVATE -Wl,-rpath=${OPENVINO_LIB_PATH}) # Install OpenVINO backend library to the lib directory install(TARGETS openvino_backend DESTINATION lib) - diff --git a/backends/openvino/partitioner.py b/backends/openvino/partitioner.py index 8e621f56508..f07be7c8410 100644 --- a/backends/openvino/partitioner.py +++ b/backends/openvino/partitioner.py @@ -7,6 +7,7 @@ from typing import Callable, final, List, Optional, Tuple import torch +import torch.fx as fx from executorch.backends.openvino.preprocess import OpenvinoBackend from executorch.exir.backend.backend_details import CompileSpec from executorch.exir.backend.partitioner import ( @@ -15,12 +16,12 @@ PartitionResult, ) from executorch.exir.backend.utils import tag_constant_data +from openvino.frontend.pytorch.torchdynamo.op_support import OperatorSupport from torch.export.exported_program import ExportedProgram from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner from torch.fx.passes.operator_support import OperatorSupportBase -import torch.fx as fx -from openvino.frontend.pytorch.torchdynamo.op_support import OperatorSupport + class OpenvinoOperatorsSupport(OperatorSupportBase): @@ -44,10 +45,10 @@ def is_node_supported(self, _, node: torch.fx.Node) -> bool: options = [] op_type = node.target.__name__ supported_ops = OperatorSupport(options)._support_dict - if (op_type == "getitem"): + if op_type == "getitem": return True - if ("torch.ops." + str(op_type) in supported_ops): + if "torch.ops." + str(op_type) in supported_ops: return True else: print("Op not supported: ", "torch.ops." + str(op_type)) @@ -58,7 +59,7 @@ def is_node_supported(self, _, node: torch.fx.Node) -> bool: ) return False - return False + return False @final @@ -88,13 +89,12 @@ def ops_to_not_decompose( return (ops_not_decompose, None) def partition(self, exported_program: ExportedProgram) -> PartitionResult: - options = {} gm = fx.symbolic_trace(exported_program.graph_module) partitioner = CapabilityBasedPartitioner( exported_program.graph_module, OpenvinoOperatorsSupport(self._op_types_to_skip, self._op_names_to_skip), - allows_single_node_partition=True + allows_single_node_partition=True, ) partition_list = partitioner.propose_partitions() diff --git a/backends/openvino/preprocess.py b/backends/openvino/preprocess.py index 6380d0d3b78..6224debc440 100644 --- a/backends/openvino/preprocess.py +++ b/backends/openvino/preprocess.py @@ -7,7 +7,7 @@ import contextlib import struct -from typing import final, List, cast +from typing import cast, final, List import torch from executorch.exir.backend.backend_details import ( @@ -18,8 +18,6 @@ from executorch.exir.backend.compile_spec_schema import CompileSpec from openvino.frontend.pytorch.torchdynamo.compile import openvino_compile -SKIP_COMPILE_SPEC_KEYS = {"ImportForever"} - @final class OpenvinoBackend(BackendDetails): @@ -34,8 +32,8 @@ def preprocess( output_names = edge_program.graph_signature.user_outputs args = [] for node in edge_program.graph.nodes: - if (node.target in input_names): - args.append( node.meta["val"]) + if node.target in input_names: + args.append(node.meta["val"]) input_shapes = [] output_shapes = [] diff --git a/backends/openvino/runtime/OpenvinoBackend.cpp b/backends/openvino/runtime/OpenvinoBackend.cpp index 95d85445f7e..890941e8c89 100644 --- a/backends/openvino/runtime/OpenvinoBackend.cpp +++ b/backends/openvino/runtime/OpenvinoBackend.cpp @@ -7,8 +7,8 @@ */ #include -#include #include +#include #include @@ -39,143 +39,159 @@ namespace backends { namespace openvino { OpenvinoBackend::OpenvinoBackend() { - if (!is_available()) { - //ET_LOG(Error, "OpenVINO runtime is not available. Initialization failed."); - throw std::runtime_error("OpenVINO runtime not available"); - } + if (!is_available()) { + // ET_LOG(Error, "OpenVINO runtime is not available. Initialization + // failed."); + throw std::runtime_error("OpenVINO runtime not available"); + } - //ET_LOG(Info, "OpenVINO runtime successfully verified and initialized."); + // ET_LOG(Info, "OpenVINO runtime successfully verified and initialized."); } bool OpenvinoBackend::is_available() const { - try { - // Create an OpenVINO Core object to verify runtime availability - ov::Core core; - - // Check if at least one device is available - auto devices = core.get_available_devices(); - if (!devices.empty()) { - return true; // OpenVINO is available - } - } catch (const std::exception& e) { - // Log the exception if OpenVINO runtime is not available - ET_LOG(Error, "OpenVINO is not available: %s", e.what()); - } catch (...) { - // Handle any unexpected errors - ET_LOG(Error, "OpenVINO availability check failed due to an unknown error."); - } + try { + // Create an OpenVINO Core object to verify runtime availability + ov::Core core; - return false; // OpenVINO is not available + // Check if at least one device is available + auto devices = core.get_available_devices(); + if (!devices.empty()) { + return true; // OpenVINO is available + } + } catch (const std::exception& e) { + // Log the exception if OpenVINO runtime is not available + ET_LOG(Error, "OpenVINO is not available: %s", e.what()); + } catch (...) { + // Handle any unexpected errors + ET_LOG( + Error, "OpenVINO availability check failed due to an unknown error."); + } + + return false; // OpenVINO is not available } Result OpenvinoBackend::init( BackendInitContext& context, FreeableBuffer* processed, ArrayRef compile_specs) const { + ET_LOG(Info, "OpenvinoBackend::init %p", processed->data()); - ET_LOG(Info, "OpenvinoBackend::init %p", processed->data()); + ov::Core core; + const char* data_ptr = static_cast(processed->data()); + size_t data_size = processed->size(); - ov::Core core; - const char* data_ptr = static_cast(processed->data()); - size_t data_size = processed->size(); + // Copy data to a string or vector + std::string data_string(data_ptr, data_size); - // Copy data to a string or vector - std::string data_string(data_ptr, data_size); + // Wrap the data in a stream + std::istringstream compiled_stream(data_string); - // Wrap the data in a stream - std::istringstream compiled_stream(data_string); + auto device = "CPU"; + // Get the device value, if provided in compile sepcs + for (auto& compile_spec : compile_specs) { + if (std::strcmp(compile_spec.key, "device") == 0) + device = static_cast(compile_spec.value.buffer); + } - // Import the model - auto compiled_model = core.import_model(compiled_stream, "CPU"); + // Import the model + auto compiled_model = core.import_model(compiled_stream, device); - // Allocate an infer request - std::shared_ptr infer_request = std::make_shared(compiled_model.create_infer_request()); + // Allocate an infer request + std::shared_ptr infer_request = + std::make_shared(compiled_model.create_infer_request()); - // Allocate execution handle - MemoryAllocator* allocator = context.get_runtime_allocator(); - ExecutionHandle* handle = ET_ALLOCATE_INSTANCE_OR_RETURN_ERROR(allocator, ExecutionHandle); - handle->compiled_model = std::make_shared(compiled_model); - handle->infer_request = infer_request; + // Allocate execution handle + MemoryAllocator* allocator = context.get_runtime_allocator(); + ExecutionHandle* handle = + ET_ALLOCATE_INSTANCE_OR_RETURN_ERROR(allocator, ExecutionHandle); + handle->compiled_model = std::make_shared(compiled_model); + handle->infer_request = infer_request; - return handle; + return handle; } Error OpenvinoBackend::execute( BackendExecutionContext& context, DelegateHandle* input_handle, EValue** args) const { + ExecutionHandle* execution_handle = (ExecutionHandle*)input_handle; - ExecutionHandle* execution_handle = (ExecutionHandle*)input_handle; + auto infer_request = execution_handle->infer_request; - auto infer_request = execution_handle->infer_request; + size_t num_inputs = infer_request->get_compiled_model().inputs().size(); + size_t num_outputs = infer_request->get_compiled_model().outputs().size(); - size_t num_inputs = infer_request->get_compiled_model().inputs().size(); - size_t num_outputs = infer_request->get_compiled_model().outputs().size(); + // Set inputs + for (size_t i = 0; i < num_inputs; i++) { + auto input_tensor = args[i]->toTensor(); + ov::Shape input_shape( + input_tensor.sizes().begin(), input_tensor.sizes().end()); - // Set inputs - for (size_t i = 0; i < num_inputs; i++) { - auto input_tensor = args[i]->toTensor(); - ov::Shape input_shape(input_tensor.sizes().begin(), input_tensor.sizes().end()); + // Convert input tensor to OpenVINO tensor + ov::element::Type ov_type = + convert_to_openvino_type(input_tensor.scalar_type()); + ov::Tensor ov_input_tensor( + ov_type, input_shape, input_tensor.mutable_data_ptr()); - // Convert input tensor to OpenVINO tensor - ov::element::Type ov_type = convert_to_openvino_type(input_tensor.scalar_type()); - ov::Tensor ov_input_tensor(ov_type, input_shape, input_tensor.mutable_data_ptr()); + infer_request->set_input_tensor(i, ov_input_tensor); + } - infer_request->set_input_tensor(i, ov_input_tensor); - } + // Set outputs + for (size_t i = 0; i < num_outputs; i++) { + auto output_tensor = args[num_inputs + i]->toTensor(); + ov::Shape output_shape( + output_tensor.sizes().begin(), output_tensor.sizes().end()); - // Set outputs - for (size_t i = 0; i < num_outputs; i++) { - auto output_tensor = args[num_inputs+i]->toTensor(); - ov::Shape output_shape(output_tensor.sizes().begin(), output_tensor.sizes().end()); + // Convert input tensor to OpenVINO tensor + ov::element::Type ov_type = + convert_to_openvino_type(output_tensor.scalar_type()); + ov::Tensor ov_output_tensor( + ov_type, output_shape, output_tensor.mutable_data_ptr()); - // Convert input tensor to OpenVINO tensor - ov::element::Type ov_type = convert_to_openvino_type(output_tensor.scalar_type()); - ov::Tensor ov_output_tensor(ov_type, output_shape, output_tensor.mutable_data_ptr()); - - infer_request->set_output_tensor(i, ov_output_tensor); - } + infer_request->set_output_tensor(i, ov_output_tensor); + } - // Execute the inference - infer_request->infer(); + // Execute the inference + infer_request->infer(); - return Error::Ok; + return Error::Ok; } void OpenvinoBackend::destroy(DelegateHandle* handle) const { - if (!handle) { - ET_LOG(Info, "Attempted to destroy a null handle."); - return; - } - - // Cast the handle to the appropriate type - ExecutionHandle* execution_handle = static_cast(handle); - - // Clean up resources - if (execution_handle->infer_request) { - execution_handle->infer_request.reset(); // Release the infer request - ET_LOG(Info, "Infer request successfully destroyed."); - } - - if (execution_handle->compiled_model) { - execution_handle->compiled_model.reset(); // Release the compiled model - ET_LOG(Info, "Compiled model successfully destroyed."); - } - - ET_LOG(Info, "Delegate handle destroyed successfully."); + if (!handle) { + ET_LOG(Info, "Attempted to destroy a null handle."); + return; + } + + // Cast the handle to the appropriate type + ExecutionHandle* execution_handle = static_cast(handle); + + // Clean up resources + if (execution_handle->infer_request) { + execution_handle->infer_request.reset(); // Release the infer request + ET_LOG(Info, "Infer request successfully destroyed."); + } + + if (execution_handle->compiled_model) { + execution_handle->compiled_model.reset(); // Release the compiled model + ET_LOG(Info, "Compiled model successfully destroyed."); + } + + ET_LOG(Info, "Delegate handle destroyed successfully."); } -ov::element::Type OpenvinoBackend::convert_to_openvino_type(ScalarType scalar_type) const { - switch (scalar_type) { - case ScalarType::Float: - return ov::element::f32; - case ScalarType::Int: - return ov::element::i32; - case ScalarType::Char: - return ov::element::i8; - default: - throw std::runtime_error("Unsupported scalar type"); - } +ov::element::Type OpenvinoBackend::convert_to_openvino_type( + ScalarType scalar_type) const { + switch (scalar_type) { + case ScalarType::Float: + return ov::element::f32; + case ScalarType::Int: + return ov::element::i32; + case ScalarType::Char: + return ov::element::i8; + default: + throw std::runtime_error("Unsupported scalar type"); + } } } // namespace openvino @@ -185,7 +201,5 @@ ov::element::Type OpenvinoBackend::convert_to_openvino_type(ScalarType scalar_ty namespace { auto backend = executorch::backends::openvino::OpenvinoBackend(); executorch::runtime::Backend backend_id{"OpenvinoBackend", &backend}; -static auto registered = executorch::runtime::register_backend(backend_id); +static auto registered = executorch::runtime::register_backend(backend_id); } // namespace - - diff --git a/examples/openvino/CMakeLists.txt b/examples/openvino/CMakeLists.txt index 10638a7b5f7..bce54af1b4a 100644 --- a/examples/openvino/CMakeLists.txt +++ b/examples/openvino/CMakeLists.txt @@ -94,5 +94,3 @@ set_target_properties(openvino_executor_runner PROPERTIES INSTALL_RPATH "$ORIGIN get_filename_component( EXECUTORCH_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE ) - - diff --git a/examples/openvino/aot/README.md b/examples/openvino/aot/README.md index 884ed55849f..adfed35eefc 100644 --- a/examples/openvino/aot/README.md +++ b/examples/openvino/aot/README.md @@ -113,5 +113,3 @@ python aot_openvino_compiler.py --suite timm --model vgg16 --input_shape [1, 3, - **Unsupported Input Shape**: Ensure `--input_shape` is provided as a valid list or tuple. - - diff --git a/examples/openvino/aot/aot_openvino_compiler.py b/examples/openvino/aot/aot_openvino_compiler.py index b7bf63a386a..17e2623d14f 100644 --- a/examples/openvino/aot/aot_openvino_compiler.py +++ b/examples/openvino/aot/aot_openvino_compiler.py @@ -206,7 +206,9 @@ def main( lowered_module: EdgeProgramManager = to_edge_transform_and_lower(aten_dialect, partitioner=[OpenvinoPartitioner(compile_spec),]) # Apply backend-specific passes - exec_prog = lowered_module.to_executorch(config=executorch.exir.ExecutorchBackendConfig()) + exec_prog = lowered_module.to_executorch( + config=executorch.exir.ExecutorchBackendConfig() + ) # Serialize and save it to a file model_file_name = f"{model_name}_{'int8' if quantize else 'fp32'}.pte" @@ -228,6 +230,7 @@ def main( print(f"acc@1: {acc_top1}") + if __name__ == "__main__": # Argument parser for dynamic inputs parser = argparse.ArgumentParser(description="Export models with executorch.")