[TensorRT EP] Fix bug for handling outer scope values in GetCapability (

microsoft#18342) The issues found in yolov3, tiny-yolov3 etc where it has control flow ops. Two modifications: 1. In GetCapability/GetSupporedtList, only if the newly built graph has control flow op as well as it has parent node, it needs to handle outer scope values before calling graph.Resolve(). 2. Two graph/subgraphs has the chance to have the same graph->Name(). Add a function to get the unique graph name.
kleiti · Mar 22, 2024 · 1c8bf5d · 1c8bf5d
1 parent bb0deb0
commit 1c8bf5d
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 43 deletions.
diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
@@ -1562,7 +1562,9 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect
           }
         }
 
-        if (has_control_flow_op) {
+        // Only if the newly built graph has control flow op as well as it has parent node,
+        // it needs to handle outer scope values before calling graph.Resolve().
+        if (has_control_flow_op && graph.ParentNode()) {
           LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Handle outer scope values for the subgraph " << graph_build.Name();
           BuildSubGraphContext(graph_build);
           SetGraphOuterScopeValuesAndInputs(graph_build, graph.GetGraph());

diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_helper.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_helper.cc
@@ -3,10 +3,36 @@
 
 #include "core/providers/shared_library/provider_api.h"
 #include "tensorrt_execution_provider.h"
+#include "core/framework/murmurhash3.h"
 #include <iostream>
 
 namespace onnxruntime {
 
+namespace {
+// Get unique graph name based on graph's name and all nodes' name
+std::string GetUniqueGraphName(const Graph& graph) {
+  HashValue model_hash = 0;
+  uint32_t hash[4] = {0, 0, 0, 0};
+
+  auto hash_str = [&hash](const std::string& str) {
+    MurmurHash3::x86_128(str.data(), gsl::narrow_cast<int32_t>(str.size()), hash[0], &hash);
+  };
+
+  // Hash all nodes' name
+  for (int i = 0; i < graph.MaxNodeIndex(); ++i) {
+    auto node = graph.GetNode(i);
+    if (node == nullptr) {
+      continue;
+    }
+    hash_str(node->Name());
+  }
+
+  model_hash = hash[0] | (uint64_t(hash[1]) << 32);
+
+  return graph.Name() + "_" + std::to_string(model_hash);
+}
+}  // namespace
+
 // The newly-built graph has not yet being resolved by Graph::Resolve(), so we can't leverage
 // Graph::ResolveContext::IsInputInitializerOrOutput(). We have to implement this fuction again.
 bool TensorrtExecutionProvider::IsInputInitializerOrOutput(const Graph& graph,
@@ -31,10 +57,11 @@ bool TensorrtExecutionProvider::IsOuterScopeValue(const Graph& graph,
 // Graph::ResolveContext::IsLocalValue(). We have to implement this function again.
 bool TensorrtExecutionProvider::IsLocalValue(const Graph& graph,
                                              const std::string& name) const {
-  if (subgraph_context_map_.find(graph.Name()) == subgraph_context_map_.end()) {
+  std::string unique_graph_name = GetUniqueGraphName(graph);
+  if (subgraph_context_map_.find(unique_graph_name) == subgraph_context_map_.end()) {
     return false;
   }
-  SubGraphContext* context = subgraph_context_map_.at(graph.Name()).get();
+  SubGraphContext* context = subgraph_context_map_.at(unique_graph_name).get();
   return context->output_args.find(name) != context->output_args.cend() ||
          context->inputs_and_initializers.find(name) != context->inputs_and_initializers.cend();
 }
@@ -59,13 +86,15 @@ void TensorrtExecutionProvider::BuildSubGraphContext(const Graph& graph) const {
     }
   }
 
+  std::string unique_graph_name = GetUniqueGraphName(graph);
+
   // Subgraph context has been built before, no need to do it again
-  if (subgraph_context_map_.find(graph.Name()) != subgraph_context_map_.end()) {
+  if (subgraph_context_map_.find(unique_graph_name) != subgraph_context_map_.end()) {
     return;
   }
 
-  subgraph_context_map_.emplace(graph.Name(), std::make_unique<SubGraphContext>());
-  SubGraphContext* context = subgraph_context_map_.at(graph.Name()).get();
+  subgraph_context_map_.emplace(unique_graph_name, std::make_unique<SubGraphContext>());
+  SubGraphContext* context = subgraph_context_map_.at(unique_graph_name).get();
 
   // Collect all nodes' outputs and nodes' name
   for (int i = 0; i < graph.MaxNodeIndex(); ++i) {
@@ -138,13 +167,14 @@ void TensorrtExecutionProvider::SetGraphOuterScopeValuesAndInputs(Graph& graph_b
     while (top_level_graph->MutableParentGraph()) {
       top_level_graph = top_level_graph->MutableParentGraph();
     }
-    if (subgraph_context_map_.find(top_level_graph->Name()) == subgraph_context_map_.end()) {
+    std::string unique_graph_name = GetUniqueGraphName(*top_level_graph);
+    if (subgraph_context_map_.find(unique_graph_name) == subgraph_context_map_.end()) {
       LOGS_DEFAULT(ERROR) << "[TensorRT EP] Can't find top-level graph context. \
                               Please check BuildSubGraphContext() has built the graph context correctly.";
       return;
     }
 
-    SubGraphContext* context = subgraph_context_map_.at(top_level_graph->Name()).get();
+    SubGraphContext* context = subgraph_context_map_.at(unique_graph_name).get();
 
     LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Subgraph name is " << graph_build.Name();
     LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Its parent node is " << graph.ParentNode()->Name();
@@ -197,12 +227,13 @@ void TensorrtExecutionProvider::SetGraphOuterScopeValuesAndInputs(Graph& graph_b
 void TensorrtExecutionProvider::SetAllGraphInputs(Graph& graph) const {
   // If ORT TRT doesn't manully set graph input in TensorrtExecutionProvider::SetGraphOuterScopeValuesAndInputs(),
   // Graph::Resolve() will help set graph inputs in Graph::SetGraphInputsOutputs(), so no need to set graph inputs here.
-  if (subgraph_context_map_.find(graph.Name()) == subgraph_context_map_.end() ||
-      subgraph_context_map_[graph.Name()].get()->manually_added_graph_inputs.size() == 0) {
+  std::string unique_graph_name = GetUniqueGraphName(graph);
+  if (subgraph_context_map_.find(unique_graph_name) == subgraph_context_map_.end() ||
+      subgraph_context_map_[unique_graph_name].get()->manually_added_graph_inputs.size() == 0) {
     return;
   }
 
-  SubGraphContext* context = subgraph_context_map_[graph.Name()].get();
+  SubGraphContext* context = subgraph_context_map_[unique_graph_name].get();
   std::vector<const NodeArg*> graph_inputs_including_initializers;
   std::unordered_set<std::string> graph_inputs_including_initializers_set;
 

diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+#include <iostream>
 #include <iterator>
 #include <gtest/gtest.h>
 
@@ -410,7 +411,7 @@ ::std::vector<::std::basic_string<ORTCHAR_T>> GetParameterStrings() {
   // If an EP doesn't have any CI build pipeline, then there is no need to specify any opset.
 #ifdef USE_TENSORRT
   // tensorrt: only enable opset 14 to 17 of onnx tests
-  provider_names[provider_name_tensorrt] = {opset14, opset15, opset16, opset17};
+  provider_names[provider_name_tensorrt] = {opset12, opset14, opset15, opset16, opset17};
 #endif
 #ifdef USE_MIGRAPHX
   provider_names[provider_name_migraphx] = {opset7, opset8, opset9, opset10, opset11, opset12, opset13, opset14, opset15, opset16, opset17, opset18};
@@ -592,37 +593,10 @@ ::std::vector<::std::basic_string<ORTCHAR_T>> GetParameterStrings() {
                                                    ORT_TSTR("mul_uint8"),
                                                    ORT_TSTR("div_uint8")};
   static const ORTCHAR_T* tensorrt_disabled_tests[] = {
-      ORT_TSTR("udnie"),
-      ORT_TSTR("rain_princess"),
-      ORT_TSTR("pointilism"),
-      ORT_TSTR("mosaic"),
-      ORT_TSTR("LSTM_Seq_lens_unpacked"),
-      ORT_TSTR("cgan"),
-      ORT_TSTR("candy"),
-      ORT_TSTR("tinyyolov3"),
-      ORT_TSTR("yolov3"),
-      ORT_TSTR("mlperf_ssd_resnet34_1200"),
-      ORT_TSTR("mlperf_ssd_mobilenet_300"),
-      ORT_TSTR("mask_rcnn"),
-      ORT_TSTR("faster_rcnn"),
-      ORT_TSTR("fp16_shufflenet"),
-      ORT_TSTR("fp16_inception_v1"),
-      ORT_TSTR("fp16_tiny_yolov2"),
-      ORT_TSTR("tf_inception_v3"),
-      ORT_TSTR("tf_mobilenet_v1_1.0_224"),
-      ORT_TSTR("tf_mobilenet_v2_1.0_224"),
-      ORT_TSTR("tf_mobilenet_v2_1.4_224"),
-      ORT_TSTR("tf_resnet_v1_101"),
-      ORT_TSTR("tf_resnet_v1_152"),
-      ORT_TSTR("tf_resnet_v1_50"),
-      ORT_TSTR("tf_resnet_v2_101"),
-      ORT_TSTR("tf_resnet_v2_152"),
-      ORT_TSTR("tf_resnet_v2_50"),
-      ORT_TSTR("convtranspose_1d"),
-      ORT_TSTR("convtranspose_3d"),
-      ORT_TSTR("conv_with_strides_and_asymmetric_padding"),
-      ORT_TSTR("conv_with_strides_padding"),
-      ORT_TSTR("size")  // INVALID_ARGUMENT: Cannot find binding of given name: x
+      ORT_TSTR("YOLOv3-12"),           // needs to run symbolic shape inference shape first
+      ORT_TSTR("SSD-MobilenetV1-12"),  // symbolic shape inference shape error
+      ORT_TSTR("SSD"),                 // needs to run symbolic shape inference shape first
+      ORT_TSTR("size")                 // INVALID_ARGUMENT: Cannot find binding of given name: x
   };
   std::vector<std::basic_string<ORTCHAR_T>> paths;