feat(ModelZoo): rework prebuilt assets management

ModelZoo artifacts changed from a single archive to one archive per model/backend combination. It allows users to only download needed archives, but prevents keeping the current design. Change from all models being handled by the "neural_networks_provider" package to models being downloaded by packages that need them. Leverage the newly added versioning of the prebuilt models. Issue-Id: SCM-3999 Signed-off-by: Ambroise Vincent <[email protected]> Change-Id: I1df9007f5bf446a8b50e38c4fd98e9e3a8d2550f
autowarefoundation · Sep 15, 2022 · 862c66a · 862c66a
1 parent 5d37181
commit 862c66a
Show file tree

Hide file tree

Showing 16 changed files with 147 additions and 253 deletions.
diff --git a/common/neural_networks_provider/CMakeLists.txt b/common/neural_networks_provider/CMakeLists.txt
@@ -18,46 +18,5 @@ project(neural_networks_provider)
 find_package(autoware_cmake REQUIRED)
 autoware_package()
 
-set(SUPPORTED_ARCH "aarch64" "x86_64")
-set(USER_DIR ${CMAKE_CURRENT_SOURCE_DIR}/user/${CMAKE_SYSTEM_PROCESSOR})
-
-set(PREFIX_DIR ${CMAKE_BINARY_DIR}/download)
-file(MAKE_DIRECTORY ${PREFIX_DIR}/src/networks)
-
-if(CMAKE_SYSTEM_PROCESSOR IN_LIST SUPPORTED_ARCH)
-  if(DOWNLOAD_ARTIFACTS)
-    set(EXEC_DOWNLOAD TRUE)
-  else()
-    set(EXEC_DOWNLOAD FALSE)
-    message(WARNING "Skipped download (enable by setting DOWNLOAD_ARTIFACTS)")
-  endif()
-else()
-  set(EXEC_DOWNLOAD FALSE)
-  message(WARNING "Skipped download (unsupported architecture)")
-endif()
-
-add_custom_target(artifact_download ALL
-  COMMAND "EXEC_DOWNLOAD=${EXEC_DOWNLOAD}"
-          "PREFIX_DIR=${PREFIX_DIR}"
-          "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}"
-          "${CMAKE_CURRENT_SOURCE_DIR}/download_neural_networks.sh"
-)
-
-# Install the pre-compiled networks
-install(
-  DIRECTORY ${PREFIX_DIR}/src/networks
-  DESTINATION share/${PROJECT_NAME}/
-  USE_SOURCE_PERMISSIONS
-)
-
-# Install user-provided networks over the pre-compiled ones
-if(EXISTS ${USER_DIR})
-  install(
-    DIRECTORY ${USER_DIR}/
-    DESTINATION share/${PROJECT_NAME}/networks/
-    USE_SOURCE_PERMISSIONS
-  )
-endif()
-
 list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "neural_networks_provider-extras.cmake")
 ament_auto_package()
diff --git a/common/neural_networks_provider/design/neural_networks-design.md b/common/neural_networks_provider/design/neural_networks-design.md
@@ -6,7 +6,7 @@
 <!-- Things to consider:
     - Why did we implement this feature? -->
 
-This package provides pre-compiled neural networks to packages using them for their inference.
+This package provides a utility to get pre-compiled neural networks to packages using them for their inference.
 
 ## Design
 
@@ -16,40 +16,58 @@ This package provides pre-compiled neural networks to packages using them for th
 
 The neural networks are compiled as part of the
 [Model Zoo](https://github.com/autowarefoundation/modelzoo/) CI pipeline and saved to an S3 bucket.
-They are downloaded and installed when this package is built when the `DOWNLOAD_ARTIFACTS` variable
-is set.
+This package exports cmake variables and functions for ease of access to those neural networks.
 
-The user can provide its own compiled networks to be made available through this package, or to
-overwrite the pre-compiled ones by creating a `user` directory and using the same directory
-structure.
+The `get_neural_network` function creates an abstraction for the artifact management.
+The artifacts are saved under the source directory of the package making use of the function; under "data/".
+Priority is given to user-provided files, under "data/user/${MODEL_NAME}/".
+If there are no user-provided files, the function tries to reuse previously-downloaded artifacts.
+If there are no previously-downloaded artifacts, and if the `DOWNLOAD_ARTIFACTS` cmake variable is set, they will be downloaded from the bucket.
+Otherwise, nothing happens.
 
-The structure is as follow:
+The structure inside of the source directory of the package making use of the function is as follow:
 
 ```{text}
 .
-├── ${ARCH 1}
-│   ├── ${MODEL 1}
-│   │   ├── ${BACKEND 1}
-│   │   │   ├── deploy_graph.json
-│   │   │   ├── deploy_lib.so
-│   │   │   ├── deploy_param.params
-│   │   │   └── inference)_engine_tvm_config.hpp
-│   │   └── ${BACKEND ...}
+├── data
+│   ├── downloads
+│   │   ├── ${MODEL 1}-${ARCH 1}-{BACKEND 1}-{VERSION 1}.tar.gz
+│   │   ├── ...
+│   │   └── ${MODEL ...}-${ARCH ...}-{BACKEND ...}-{VERSION ...}.tar.gz
+│   ├── models
+│   │   ├── ${MODEL 1}
+│   │   │   ├── ...
+│   │   │   └── inference_engine_tvm_config.hpp
+│   │   ├── ...
+│   │   └── ${MODEL ...}
 │   │       └── ...
-│   └── ${MODEL ...}
-│       └── ...
-└── ${ARCH ...}
-    └── ...
+│   └── user
+│       ├── ${MODEL 1}
+│       │   ├── deploy_graph.json
+│       │   ├── deploy_lib.so
+│       │   ├── deploy_param.params
+│       │   └── inference_engine_tvm_config.hpp
+│       ├── ...
+│       └── ${MODEL ...}
+│           └── ...
 ```
 
-The pre-compiled networks are downloaded at build time when a new revision is available, provided
-that `DOWNLOAD_ARTIFACTS` is set (see Inputs).
+The `inference_engine_tvm_config.hpp` file needed for compilation by dependent packages is made available under "data/models/${MODEL_NAME}/inference_engine_tvm_config.hpp".
+Dependent packages can use the cmake `add_dependencies` function with the name provided in the `DEPENDENCY` output parameter of `get_neural_network` to ensure this file is created before it gets used.
+
+The other `deploy_*` files are installed to "models/${MODEL_NAME}/" under the `share` directory of the package.
+
+The target version to be downloaded can be overwritten by setting the `MODELZOO_VERSION` cmake variable.
 
 ### Assumptions / Known limits
 
 <!-- Required -->
 
-An internet connection is required at build time.
+If several packages make use of the same neural network, it will be downloaded once per package.
+
+In case a requested artifact doesn't exist in the S3 bucket, the error message from ExternalProject is not explicit enough for the user to understand what went wrong.
+
+In case the user manually sets `MODELZOO_VERSION` to "latest", the archive will not be re-downloaded when it gets updated in the S3 bucket (it is not a problem for tagged versions as they are not expected to be updated).
 
 ### Inputs / Outputs / API
 
@@ -59,21 +77,21 @@ An internet connection is required at build time.
 
 Inputs:
 
-- `DOWNLOAD_ARTIFACTS` variable needs to be set to enable downloading the artifacts
+- `DOWNLOAD_ARTIFACTS` cmake variable; needs to be set to enable downloading the artifacts
+- `MODELZOO_VERSION` cmake variable; can be used to overwrite the default target version of downloads
 
 Outputs:
 
-- `neural_networks_provider_NETWORKS_DIR` variable containing the path to the root directory of the networks
-- `neural_networks_provider_NAMES` variable containing the list of available networks
+- `get_neural_network` cmake function; can be used to get a neural network compiled for a specific backend
 
 API: none
 
 ### Error detection and handling
 
 <!-- Required -->
 
-Building the package will not fail.
-If no models are available for the target architecture, then a log message is displayed.
+The `DEPENDENCY` argument of `get_neural_network` can be checked for the outcome of the function.
+It is an empty string when the neural network couldn't be made available.
 
 ## Security considerations
 

diff --git a/common/neural_networks_provider/download_neural_networks.sh b/common/neural_networks_provider/download_neural_networks.sh
diff --git a/common/neural_networks_provider/neural_networks_provider-extras.cmake b/common/neural_networks_provider/neural_networks_provider-extras.cmake
@@ -12,21 +12,73 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-normalize_path(neural_networks_provider_NETWORKS_DIR
-  "${neural_networks_provider_DIR}/../networks"
-)
-if(NOT IS_DIRECTORY "${neural_networks_provider_NETWORKS_DIR}")
-  message(WARNING "Package 'neural_networks_provider' exports the directory '${neural_networks_provider_NETWORKS_DIR}' which doesn't exist")
-endif()
+# Get user-provided variables
+set(DOWNLOAD_ARTIFACTS OFF CACHE BOOL "enable artifacts download")
+set(MODELZOO_VERSION "1.3.0-20220902" CACHE STRING "targeted ModelZoo version")
 
-macro(_subdirlist result dir)
-  file(GLOB children RELATIVE ${dir} ${dir}/*)
-  set(${result} "")
-  foreach(child ${children})
-    if(IS_DIRECTORY ${dir}/${child})
-      list(APPEND ${result} ${child})
+#
+# Download the selected neural network if it is not already present on disk.
+# Make inference_engine_tvm_config.hpp available under "data/models/${MODEL_NAME}/".
+# Install the TVM artifacts to "share/${PROJECT_NAME}/models/".
+# Return the name of the custom target in the DEPENDENCY parameter.
+#
+# :param MODEL_NAME: the name of the targeted neural network
+# :type MODEL_NAME: string
+# :param MODEL_BACKEND: the name of the targeted backend
+# :type MODEL_BACKEND: string
+# :param DEPENDENCY: output parameter; name of the ExternalProject top level target
+# :type DEPENDENCY: string
+#
+function(get_neural_network MODEL_NAME MODEL_BACKEND DEPENDENCY)
+  set(DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}/data)
+  set(EXTERNALPROJECT_NAME ${MODEL_NAME}_${MODEL_BACKEND})
+
+  # Prioritize user-provided models.
+  if(IS_DIRECTORY "${DATA_PATH}/user/${MODEL_NAME}")
+    message(STATUS "Using user-provided model from ${DATA_PATH}/user/${MODEL_NAME}")
+    file(REMOVE_RECURSE "${DATA_PATH}/models/${MODEL_NAME}/")
+    configure_file(
+      "${DATA_PATH}/user/${MODEL_NAME}/inference_engine_tvm_config.hpp"
+      "${DATA_PATH}/models/${MODEL_NAME}/inference_engine_tvm_config.hpp"
+      COPYONLY
+    )
+    install(
+      DIRECTORY "${DATA_PATH}/user/${MODEL_NAME}"
+      DESTINATION "share/${PROJECT_NAME}/models/"
+      USE_SOURCE_PERMISSIONS
+    )
+  else()
+    set(ARCHIVE_NAME "${MODEL_NAME}-${CMAKE_SYSTEM_PROCESSOR}-${MODEL_BACKEND}-${MODELZOO_VERSION}.tar.gz")
+
+    # Use previously-downloaded archives if available.
+    if(EXISTS "${DATA_PATH}/downloads/${ARCHIVE_NAME}")
+      set(URL "${DATA_PATH}/downloads/${ARCHIVE_NAME}")
+    elseif(DOWNLOAD_ARTIFACTS)
+      message(STATUS "Downloading ${ARCHIVE_NAME} ...")
+      set(URL "https://autoware-modelzoo.s3.us-east-2.amazonaws.com/models/${MODELZOO_VERSION}/${ARCHIVE_NAME}")
+    else()
+      message(WARNING "Skipped download for ${MODEL_NAME} (enable by setting DOWNLOAD_ARTIFACTS)")
+      set(${DEPENDENCY} "" PARENT_SCOPE)
+      return()
     endif()
-  endforeach()
-endmacro()
 
-_subdirlist(neural_networks_provider_NAMES ${neural_networks_provider_NETWORKS_DIR})
+    include(ExternalProject)
+    externalproject_add(${EXTERNALPROJECT_NAME}
+      DOWNLOAD_DIR "${DATA_PATH}/downloads"
+      SOURCE_DIR "${DATA_PATH}/models/${MODEL_NAME}"
+      URL ${URL}
+      CONFIGURE_COMMAND ""
+      BUILD_COMMAND ""
+      BUILD_BYPRODUCTS "${DATA_PATH}/models/${MODEL_NAME}/inference_engine_tvm_config.hpp"
+      INSTALL_COMMAND ""
+    )
+    install(
+      DIRECTORY "${DATA_PATH}/models/${MODEL_NAME}"
+      DESTINATION "share/${PROJECT_NAME}/models/"
+      USE_SOURCE_PERMISSIONS
+    )
+  endif()
+
+  set(${DEPENDENCY} ${EXTERNALPROJECT_NAME} PARENT_SCOPE)
+
+endfunction()
diff --git a/common/neural_networks_provider/package.xml b/common/neural_networks_provider/package.xml
@@ -8,9 +8,6 @@
   <license>Apache 2.0</license>
 
   <buildtool_depend>ament_cmake_auto</buildtool_depend>
-  <buildtool_depend>coreutils</buildtool_depend>
-  <buildtool_depend>curl</buildtool_depend>
-  <buildtool_depend>tar</buildtool_depend>
 
   <build_depend>autoware_cmake</build_depend>
 

diff --git a/common/tvm_utility/.gitignore b/common/tvm_utility/.gitignore
@@ -1,3 +1,2 @@
-libs
-*.o
-*.so
+artifacts/**/*.jpg
+data/