intel · suryasidd · Jun 17, 2020 · Jun 17, 2020 · Jun 17, 2020 · Jun 17, 2020
diff --git a/BUILD.md b/BUILD.md
diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc
@@ -253,6 +253,13 @@ void BackendManager::Compute(Ort::CustomOpApi api, OrtKernelContext* context) {
     std::vector<std::vector<int64_t>> tensor_shapes = GetInputTensorShapes(api, context);
     auto key = MakeMapKeyString(tensor_shapes, subgraph_context_.device_id);
 
+    if(subgraph_context_.device_id == "MYRIAD"){
+      for(size_t i = 0; i < subgraph_context_.input_indexes.size(); i++){
+        if(tensor_shapes[i].size() != 4)
+          subgraph_context_.set_vpu_config = true;
+      }
+    }
+
     std::shared_ptr<IBackend> dynamic_backend;
     auto search = backend_map_.find(key);
     if (search == backend_map_.end()) {

diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc
@@ -217,6 +217,30 @@ GetOutputTensors(Ort::CustomOpApi& ort, OrtKernelContext* context, size_t batch_
   return output_tensors;
 }
 
+int GetFirstAvailableDevice(GlobalContext& global_context){
+
+  int i = 0;
+  //Get the first available VAD-M device and set the device to busy
+  while(i < 8){
+    bool device = global_context.deviceAvailableList[i];
+    if(device){
+      global_context.deviceAvailableList[i] = false;
+      break;
+    }
+    i++;
+  }
+  //If all of the devices are busy, assign the first device and
+  //make all remaining devices free
+  if(i == 8){
+    i = 0;
+    global_context.deviceAvailableList[i] = false;
+    for(int j = 1; j < 8; j++){
+      global_context.deviceAvailableList[j] = true;
+    }
+  }
+  return i;
+}
+
 }  // namespace backend_utils
 }  // namespace openvino_ep
 }  // namespace onnxruntime
diff --git a/onnxruntime/core/providers/openvino/backend_utils.h b/onnxruntime/core/providers/openvino/backend_utils.h
@@ -6,6 +6,7 @@
 #include <inference_engine.hpp>
 
 #include "core/session/onnxruntime_cxx_api.h"
+#include "contexts.h"
 
 namespace onnxruntime {
 namespace openvino_ep {
@@ -23,6 +24,8 @@ void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto,
   CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, std::string device_id,
                    InferenceEngine::Precision precision);
 
+int GetFirstAvailableDevice(GlobalContext& global_context);
+
 InferenceEngine::Precision
 ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE::TypeProto& onnx_type);
 

diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc
@@ -32,7 +32,7 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
 
   // Loading model to the plugin
   std::map<std::string, std::string> config;
-  if(subgraph_context_.set_vpu_config){
+  if(subgraph_context_.device_id == "MYRIAD" && subgraph_context_.set_vpu_config){
     config["VPU_DETECT_NETWORK_BATCH"] = CONFIG_VALUE(NO);
   }
   try {

diff --git a/onnxruntime/core/providers/openvino/backends/vadm_backend.cc b/onnxruntime/core/providers/openvino/backends/vadm_backend.cc
@@ -16,6 +16,7 @@
 #include "../contexts.h"
 #include "../backend_utils.h"
 #include "vadm_backend.h"
+#include <vpu/hddl_plugin_config.hpp>
 
 namespace onnxruntime {
 namespace openvino_ep {
@@ -43,32 +44,65 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
   ie_cnn_network_ = CreateCNNNetwork(model_proto, subgraph_context_.device_id, subgraph_context_.precision);
 
   SetIODefs(model_proto, ie_cnn_network_);
+  std::map<std::string, std::string> config;
 
+  int i = 0;
   // Loading model to the plugin
-  InferenceEngine::ExecutableNetwork exe_network;
-  try {
-    exe_network = global_context_.ie_core.LoadNetwork(*ie_cnn_network_, "HDDL");
-  } catch (InferenceEngine::details::InferenceEngineException e) {
-    ORT_THROW(log_tag + " Exception while Loading Network for graph: " + subgraph_context_.subgraph_name + e.what());
-  } catch (...) {
-    ORT_THROW(log_tag + " Exception while Loading Network for graph " + subgraph_context_.subgraph_name);
+  //If graph is fully supported and batching is enabled, load the network onto all VPU's and infer
+  std::vector<InferenceEngine::ExecutableNetwork> exe_networks;
+  if(global_context_.is_wholly_supported_graph && subgraph_context_.enable_batching){
+    for(int j = 0; j < 8; j++){
+      InferenceEngine::ExecutableNetwork exe_network;
+      config[VPU_HDDL_CONFIG_KEY(DEVICE_TAG)] = global_context_.deviceTags[j];
+      try {
+        exe_network = global_context_.ie_core.LoadNetwork(*ie_cnn_network_, "HDDL", config);
+      } catch (InferenceEngine::details::InferenceEngineException e) {
+        ORT_THROW(log_tag + " Exception while Loading Network for graph: " + subgraph_context_.subgraph_name + e.what());
+      } catch (...) {
+        ORT_THROW(log_tag + " Exception while Loading Network for graph " + subgraph_context_.subgraph_name);
+      }
+      exe_networks.push_back(exe_network);
+    }
+    LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
+    for(size_t i = 0; i < num_inf_reqs_; i++) {
+      InferenceEngine::InferRequest::Ptr infRequest;
+      try {
+        infRequest = exe_networks[i].CreateInferRequestPtr();
+      } catch(InferenceEngine::details::InferenceEngineException e) {
+        ORT_THROW(log_tag + "Exception while creating InferRequest object: " + e.what());
+      } catch (...) {
+        ORT_THROW(log_tag + "Exception while creating InferRequest object.");
+      }
+      infer_requests_.push_back(infRequest);
+    }
+    LOGS_DEFAULT(INFO) << log_tag << "Infer Requests created: " << num_inf_reqs_ << std::endl;
   }
-  LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
-
-  // Create infer request
-  for (size_t i = 0; i < num_inf_reqs_; i++) {
-    InferenceEngine::InferRequest::Ptr infRequest;
+  //If the graph is not fully supported, need to schedule each subgraph on different VPU
+  //If batching is disabled just schedule on the first VPU
+  else {
+    i = GetFirstAvailableDevice(global_context);
+    LOGS_DEFAULT(INFO) << log_tag << "Device Tag is: " << i;
+    config[VPU_HDDL_CONFIG_KEY(DEVICE_TAG)] = global_context_.deviceTags[i];
+    InferenceEngine::ExecutableNetwork exe_network;
     try {
-      infRequest = exe_network.CreateInferRequestPtr();
+      exe_network = global_context_.ie_core.LoadNetwork(*ie_cnn_network_, "HDDL", config);
     } catch (InferenceEngine::details::InferenceEngineException e) {
+      ORT_THROW(log_tag + " Exception while Loading Network for graph: " + subgraph_context_.subgraph_name + e.what());
+    } catch (...) {
+      ORT_THROW(log_tag + " Exception while Loading Network for graph " + subgraph_context_.subgraph_name);
+    }
+    LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
+    InferenceEngine::InferRequest::Ptr infRequest;
+    try{
+      infRequest = exe_network.CreateInferRequestPtr();
+    } catch(InferenceEngine::details::InferenceEngineException e) {
       ORT_THROW(log_tag + "Exception while creating InferRequest object: " + e.what());
     } catch (...) {
       ORT_THROW(log_tag + "Exception while creating InferRequest object.");
     }
-
     infer_requests_.push_back(infRequest);
+    LOGS_DEFAULT(INFO) << log_tag << "Infer Requests created: 1" << std::endl;
   }
-  LOGS_DEFAULT(INFO) << log_tag << "Infer requests created: " << num_inf_reqs_;
 }
 
 // Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on
@@ -222,7 +256,6 @@ void VADMBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) {
     CompleteAsyncInference(ort, output_tensors, batch_slice_idx, inf_req_idx, infer_requests_, ie_cnn_network_);
   }
 
-  std::cout << "Inference successful" << std::endl;
   LOGS_DEFAULT(INFO) << log_tag << "Inference successful";
 }
 

diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h
@@ -12,6 +12,8 @@ namespace openvino_ep {
 struct GlobalContext {
   InferenceEngine::Core ie_core;
   bool is_wholly_supported_graph = false;
+  std::vector<bool> deviceAvailableList = {true, true, true, true, true, true, true, true};
+  std::vector<std::string> deviceTags = {"0", "1", "2", "3", "4", "5", "6", "7"};
 };
 
 // Holds context specific to subgraph.

diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc
@@ -61,7 +61,7 @@ int GetInputCount(const Node* node, const InitializedTensorSet& initializer_set)
   return count;
 }
 
-bool IsDimensionSupported(const Node* node, std::string device) {
+bool IsDimensionSupported(const Node* node) {
   auto node_inputs = node->InputDefs();
   size_t input_dims = 0;
   if (node_inputs[0]->Shape() == nullptr) {
@@ -85,12 +85,6 @@ bool IsDimensionSupported(const Node* node, std::string device) {
       auto axis = attributes["axis"].i();
       if (input_dims - axis != 1)
         return false;
-
-      //3D input not supported on GPU, MYRIAD and HDDL
-      if (device == "GPU" || device == "MYRIAD" || device == "HDDL") {
-        if (input_dims == 3)
-          return false;
-      }
     }
   }
   return true;
@@ -186,7 +180,7 @@ bool IsUnsupportedOp(std::string name, std::string device) {
 }
 
 // Returns true only if op is in a mode that is not currently supported
-static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer& graph_viewer, const std::string& device_id) {
+static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer& graph_viewer) {
   const auto& optype = node->OpType();
 
   const auto& initializers = graph_viewer.GetAllInitializedTensors();
@@ -217,7 +211,7 @@ static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer
     if (attributes.find("dilations") != attributes.end()) {
       return true;
     }
-    if (!IsDimensionSupported(node, device_id))
+    if (!IsDimensionSupported(node))
       return true;
   } else if (optype == "Add" || optype == "Sub" || optype == "Mul") {
     for (size_t i = 0; i < node->InputDefs().size(); i++) {
@@ -297,10 +291,10 @@ static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer
     }
     return true;
   } else if (optype == "Softmax") {
-    if (!IsDimensionSupported(node, device_id))
+    if (!IsDimensionSupported(node))
       return true;
   } else if (optype == "Unsqueeze") {
-    if (!IsDimensionSupported(node, device_id))
+    if (!IsDimensionSupported(node))
       return true;
   } else if (optype == "Pad") {
     // Pad is only supported only up to opset 10 (in opset 11 more inputs were added)
@@ -392,7 +386,7 @@ static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer
     if (ceil_attr != attributes.end() && ceil_attr->second.i() != 0) {
       return true;
     }
-    if (!IsDimensionSupported(node, device_id))
+    if (!IsDimensionSupported(node))
       return true;
   } else if (optype == "QLinearMatMul") {
     const auto& a_zero_point = node->InputDefs()[2];
@@ -599,7 +593,7 @@ static bool IsNodeSupported(const std::map<std::string, std::set<std::string>>&
   }
 
   //Check 3a
-  if (domain == kOnnxDomain && IsUnsupportedOpMode(node, graph_viewer, device_id)) {
+  if (domain == kOnnxDomain && IsUnsupportedOpMode(node, graph_viewer)) {
 #ifndef NDEBUG
     if (openvino_ep::backend_utils::IsDebugEnabled()) {
       std::cout << "Failed in unsupported op mode" << std::endl;