diff --git a/common/neural_networks_provider/CMakeLists.txt b/common/neural_networks_provider/CMakeLists.txt deleted file mode 100644 index bf4e5ec03d84d..0000000000000 --- a/common/neural_networks_provider/CMakeLists.txt +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2021-2022 Arm Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cmake_minimum_required(VERSION 3.14) -project(neural_networks_provider) - -find_package(autoware_cmake REQUIRED) -autoware_package() - -set(SUPPORTED_ARCH "aarch64" "x86_64") -set(USER_DIR ${CMAKE_CURRENT_SOURCE_DIR}/user/${CMAKE_SYSTEM_PROCESSOR}) - -set(PREFIX_DIR ${CMAKE_BINARY_DIR}/download) -file(MAKE_DIRECTORY ${PREFIX_DIR}/src/networks) - -if(CMAKE_SYSTEM_PROCESSOR IN_LIST SUPPORTED_ARCH) - if(DOWNLOAD_ARTIFACTS) - set(EXEC_DOWNLOAD TRUE) - else() - set(EXEC_DOWNLOAD FALSE) - message(WARNING "Skipped download (enable by setting DOWNLOAD_ARTIFACTS)") - endif() -else() - set(EXEC_DOWNLOAD FALSE) - message(WARNING "Skipped download (unsupported architecture)") -endif() - -add_custom_target(artifact_download ALL - COMMAND "EXEC_DOWNLOAD=${EXEC_DOWNLOAD}" - "PREFIX_DIR=${PREFIX_DIR}" - "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}" - "${CMAKE_CURRENT_SOURCE_DIR}/download_neural_networks.sh" -) - -# Install the pre-compiled networks -install( - DIRECTORY ${PREFIX_DIR}/src/networks - DESTINATION share/${PROJECT_NAME}/ - USE_SOURCE_PERMISSIONS -) - -# Install user-provided networks over the pre-compiled ones -if(EXISTS ${USER_DIR}) - install( - DIRECTORY ${USER_DIR}/ - DESTINATION share/${PROJECT_NAME}/networks/ - USE_SOURCE_PERMISSIONS - ) -endif() - -list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "neural_networks_provider-extras.cmake") -ament_auto_package() diff --git a/common/neural_networks_provider/design/neural_networks-design.md b/common/neural_networks_provider/design/neural_networks-design.md deleted file mode 100644 index 8532298f21271..0000000000000 --- a/common/neural_networks_provider/design/neural_networks-design.md +++ /dev/null @@ -1,101 +0,0 @@ -# Neural Networks Provider design {#neural-networks-provider-design} - -## Purpose / Use cases - - - - -This package provides pre-compiled neural networks to packages using them for their inference. - -## Design - - - - -The neural networks are compiled as part of the -[Model Zoo](https://github.com/autowarefoundation/modelzoo/) CI pipeline and saved to an S3 bucket. -They are downloaded and installed when this package is built when the `DOWNLOAD_ARTIFACTS` variable -is set. - -The user can provide its own compiled networks to be made available through this package, or to -overwrite the pre-compiled ones by creating a `user` directory and using the same directory -structure. - -The structure is as follow: - -```{text} -. -├── ${ARCH 1} -│ ├── ${MODEL 1} -│ │ ├── ${BACKEND 1} -│ │ │ ├── deploy_graph.json -│ │ │ ├── deploy_lib.so -│ │ │ ├── deploy_param.params -│ │ │ └── inference)_engine_tvm_config.hpp -│ │ └── ${BACKEND ...} -│ │ └── ... -│ └── ${MODEL ...} -│ └── ... -└── ${ARCH ...} - └── ... -``` - -The pre-compiled networks are downloaded at build time when a new revision is available, provided -that `DOWNLOAD_ARTIFACTS` is set (see Inputs). - -### Assumptions / Known limits - - - -An internet connection is required at build time. - -### Inputs / Outputs / API - - - - -Inputs: - -- `DOWNLOAD_ARTIFACTS` variable needs to be set to enable downloading the artifacts - -Outputs: - -- `neural_networks_provider_NETWORKS_DIR` variable containing the path to the root directory of the networks -- `neural_networks_provider_NAMES` variable containing the list of available networks - -API: none - -### Error detection and handling - - - -Building the package will not fail. -If no models are available for the target architecture, then a log message is displayed. - -## Security considerations - - - - -The pre-compiled networks are downloaded from an S3 bucket and are under threat of spoofing, -tampering and denial of service. -Spoofing is mitigated by using an https connection. -Mitigations for tampering and denial of service are left to AWS. - -The user-provided networks are installed as they are on the host system. -The user is in charge of securing the files they provide with regard to information disclosure. - -## Related issues - - - -- #226: Autoware.Auto Neural Networks Inference Architecture Design diff --git a/common/neural_networks_provider/download_neural_networks.sh b/common/neural_networks_provider/download_neural_networks.sh deleted file mode 100755 index 098da63e6f7ff..0000000000000 --- a/common/neural_networks_provider/download_neural_networks.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -eu - -# Copyright (c) 2021-2022, Arm Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The variables "EXEC_DOWNLOAD", "PREFIX_DIR" and "CMAKE_SYSTEM_PROCESSOR" are -# passed to this script by the CmakeLists.txt of the package itself. - -NETWORKS_FILE="$PREFIX_DIR"/src/networks-"$CMAKE_SYSTEM_PROCESSOR".tar.gz -S3_BUCKET=https://autoware-modelzoo.s3.us-east-2.amazonaws.com/ - -# Function to download-extract the networks artifact -download_artifact() { - curl -sS -o "$NETWORKS_FILE" "$S3_BUCKET"networks-"$CMAKE_SYSTEM_PROCESSOR".tar.gz - tar -zxvf "$NETWORKS_FILE" -C "$PREFIX_DIR"/src/networks -} - -if [ "$EXEC_DOWNLOAD" = "TRUE" ]; then - - # If the networks artifact is not present, simply download - if [ ! -f "$NETWORKS_FILE" ]; then - download_artifact - - # If the networks artifact is already present, check the MD5 hash and, if needed, re-download it - else - OLD_HASH=$(md5sum "$NETWORKS_FILE" | cut -d ' ' -f 1) - NEW_HASH=$(curl -sS "$S3_BUCKET"networks-"$CMAKE_SYSTEM_PROCESSOR".md5) - if [ "$OLD_HASH" != "$NEW_HASH" ]; then - download_artifact - fi - fi -fi diff --git a/common/neural_networks_provider/neural_networks_provider-extras.cmake b/common/neural_networks_provider/neural_networks_provider-extras.cmake deleted file mode 100644 index 2a1168f8674ce..0000000000000 --- a/common/neural_networks_provider/neural_networks_provider-extras.cmake +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2021-2022 Arm Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -normalize_path(neural_networks_provider_NETWORKS_DIR - "${neural_networks_provider_DIR}/../networks" -) -if(NOT IS_DIRECTORY "${neural_networks_provider_NETWORKS_DIR}") - message(WARNING "Package 'neural_networks_provider' exports the directory '${neural_networks_provider_NETWORKS_DIR}' which doesn't exist") -endif() - -macro(_subdirlist result dir) - file(GLOB children RELATIVE ${dir} ${dir}/*) - set(${result} "") - foreach(child ${children}) - if(IS_DIRECTORY ${dir}/${child}) - list(APPEND ${result} ${child}) - endif() - endforeach() -endmacro() - -_subdirlist(neural_networks_provider_NAMES ${neural_networks_provider_NETWORKS_DIR}) diff --git a/common/neural_networks_provider/package.xml b/common/neural_networks_provider/package.xml deleted file mode 100644 index 678fc8ebc2531..0000000000000 --- a/common/neural_networks_provider/package.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - neural_networks_provider - 1.0.0 - A wrapper package providing compiled neural networks. - Ambroise Vincent - Apache 2.0 - - ament_cmake_auto - coreutils - curl - tar - - autoware_cmake - - ament_lint_auto - autoware_lint_common - - - ament_cmake - - diff --git a/common/tvm_utility/.gitignore b/common/tvm_utility/.gitignore index ade3ffe59a47f..a09bb7234b379 100644 --- a/common/tvm_utility/.gitignore +++ b/common/tvm_utility/.gitignore @@ -1,3 +1,2 @@ -libs -*.o -*.so +artifacts/**/*.jpg +data/ diff --git a/common/tvm_utility/CMakeLists.txt b/common/tvm_utility/CMakeLists.txt index e4be23f40fca7..b49f141d9e2e4 100644 --- a/common/tvm_utility/CMakeLists.txt +++ b/common/tvm_utility/CMakeLists.txt @@ -15,37 +15,20 @@ cmake_minimum_required(VERSION 3.14) project(tvm_utility) +include("${PROJECT_NAME}-extras.cmake") + find_package(autoware_cmake REQUIRED) autoware_package() -# Configure - -set(tvm_utility_NETWORKS_DIR ${neural_networks_provider_NETWORKS_DIR}) -configure_file(tvm_utility-extras.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake @ONLY) - -foreach(network_name ${neural_networks_provider_NAMES}) - list(APPEND MODEL_INCLUDES_LIST - "#define INCLUDE <${network_name}/NETWORKS_BACKEND/inference_engine_tvm_config.hpp>" - "#include INCLUDE" - "#undef INCLUDE" - ) -endforeach() -list(JOIN MODEL_INCLUDES_LIST "\n" GENERATED_MODEL_INCLUDES) -configure_file(model_zoo.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/tvm_utility/model_zoo.hpp) - # Library set(TVM_UTILITY_NODE_LIB_HEADERS "include/${PROJECT_NAME}/pipeline.hpp" - "${CMAKE_CURRENT_BINARY_DIR}/include/tvm_utility/model_zoo.hpp" ) ament_auto_add_library(${PROJECT_NAME} SHARED ${TVM_UTILITY_NODE_LIB_HEADERS}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) -ament_export_include_directories(${tvm_utility_NETWORKS_DIR}) -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/ DESTINATION include) - if(BUILD_TESTING) # Get target backend set(${PROJECT_NAME}_BACKEND llvm CACHE STRING "${PROJECT_NAME} neural network backend") @@ -53,10 +36,8 @@ if(BUILD_TESTING) # compile each folder inside test/ as a test case find_package(ament_cmake_gtest REQUIRED) find_package(OpenCV REQUIRED) - find_package(tvm_vendor REQUIRED) set(tvm_runtime_DIR ${tvm_vendor_DIR}) find_package(tvm_runtime CONFIG REQUIRED) - include(${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake) set(TEST_ARTIFACTS "${CMAKE_CURRENT_LIST_DIR}/artifacts") file(GLOB TEST_CASES test/*) @@ -67,10 +48,11 @@ if(BUILD_TESTING) # the folder name becomes the test case name file(RELATIVE_PATH TEST_CASE_NAME ${CMAKE_CURRENT_LIST_DIR}/test ${TEST_FOLDER}) - # Test if files exist. The result is set in ${TEST_CASE_NAME}_FOUND - autoware_check_neural_network(${TEST_CASE_NAME} "${${PROJECT_NAME}_BACKEND}") + # Get neural network. + set(NN_DEPENDENCY "") + get_neural_network(${TEST_CASE_NAME} ${${PROJECT_NAME}_BACKEND} NN_DEPENDENCY) - if(${TEST_CASE_NAME}_FOUND) + if(NOT NN_DEPENDENCY STREQUAL "") if(TEST_CASE_NAME STREQUAL "yolo_v2_tiny" AND NOT EXISTS ${TEST_ARTIFACTS}/yolo_v2_tiny/test_image_0.jpg) message(WARNING "Missing image artifact for yolo_v2_tiny, skipping test") @@ -82,7 +64,8 @@ if(BUILD_TESTING) ament_target_dependencies(${TEST_CASE_NAME} "ament_index_cpp" "tvm_vendor" - "sensor_msgs") + ) + add_dependencies(${TEST_CASE_NAME} ${NN_DEPENDENCY}) target_link_libraries("${TEST_CASE_NAME}" "${OpenCV_LIBRARIES}" @@ -91,14 +74,11 @@ if(BUILD_TESTING) target_include_directories("${TEST_CASE_NAME}" SYSTEM PUBLIC "${OpenCV_INCLUDE_DIRS}" - "${tvm_vendor_INCLUDE_DIRS}" + "${tvm_utility_FOUND_INCLUDE_DIRS}" + "data/models" "include" - "${CMAKE_CURRENT_BINARY_DIR}/include" - "${tvm_utility_NETWORKS_DIR}" ) - target_compile_definitions(${TEST_CASE_NAME} PRIVATE NETWORKS_BACKEND=${${PROJECT_NAME}_BACKEND}) - # Install test-specific files if(IS_DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME}) install(DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME}/ @@ -113,5 +93,5 @@ if(BUILD_TESTING) endforeach() endif() -list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake") +list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${PROJECT_NAME}-extras.cmake") ament_auto_package() diff --git a/common/tvm_utility/design/tvm-utility-design.md b/common/tvm_utility/design/tvm-utility-design.md index 978bc0c2576f9..2e322261152c0 100644 --- a/common/tvm_utility/design/tvm-utility-design.md +++ b/common/tvm_utility/design/tvm-utility-design.md @@ -14,7 +14,7 @@ The Pipeline Class is a standardized way to write an inference pipeline. The pip the pre-processor, the inference engine and the post-processor. The TVM implementation of an inference engine stage is provided. -### Inputs / Outputs / API +### API The pre-processor and post-processor need to be implemented by the user before instantiating the pipeline. You can see example usage in this [example_pipeline](../test/yolo_v2_tiny). @@ -30,23 +30,92 @@ int main() { } ``` -#### Outputs +#### Models -- `autoware_check_neural_network` cmake macro to check if a specific network and backend combination exists - -#### Backend - -Dependent packages are expected to include `model_zoo.hpp` in order to get the TVM configuration structure of the targeted model/backend combination. -The backend used to do the inference can be specified by setting `NETWORKS_BACKEND` as a compile definition. -It defaults to `llvm`. +Dependent packages are expected to use the `get_neural_network` cmake function from this package in order to get the compiled TVM models. ### Error detection and handling `std::runtime_error` should be thrown whenever an error is encountered. It should be populated with an appropriate text error description. +### Neural Networks Provider + +This package also provides a utility to get pre-compiled neural networks to packages using them for their inference. + +The neural networks are compiled as part of the +[Model Zoo](https://github.com/autowarefoundation/modelzoo/) CI pipeline and saved to an S3 bucket. +This package exports cmake variables and functions for ease of access to those neural networks. + +The `get_neural_network` function creates an abstraction for the artifact management. +The artifacts are saved under the source directory of the package making use of the function; under "data/". +Priority is given to user-provided files, under "data/user/${MODEL_NAME}/". +If there are no user-provided files, the function tries to reuse previously-downloaded artifacts. +If there are no previously-downloaded artifacts, and if the `DOWNLOAD_ARTIFACTS` cmake variable is set, they will be downloaded from the bucket. +Otherwise, nothing happens. + +The structure inside of the source directory of the package making use of the function is as follow: + +```{text} +. +├── data +│ ├── downloads +│ │ ├── ${MODEL 1}-${ARCH 1}-{BACKEND 1}-{VERSION 1}.tar.gz +│ │ ├── ... +│ │ └── ${MODEL ...}-${ARCH ...}-{BACKEND ...}-{VERSION ...}.tar.gz +│ ├── models +│ │ ├── ${MODEL 1} +│ │ │ ├── ... +│ │ │ └── inference_engine_tvm_config.hpp +│ │ ├── ... +│ │ └── ${MODEL ...} +│ │ └── ... +│ └── user +│ ├── ${MODEL 1} +│ │ ├── deploy_graph.json +│ │ ├── deploy_lib.so +│ │ ├── deploy_param.params +│ │ └── inference_engine_tvm_config.hpp +│ ├── ... +│ └── ${MODEL ...} +│ └── ... +``` + +The `inference_engine_tvm_config.hpp` file needed for compilation by dependent packages is made available under "data/models/${MODEL_NAME}/inference_engine_tvm_config.hpp". +Dependent packages can use the cmake `add_dependencies` function with the name provided in the `DEPENDENCY` output parameter of `get_neural_network` to ensure this file is created before it gets used. + +The other `deploy_*` files are installed to "models/${MODEL_NAME}/" under the `share` directory of the package. + +The target version to be downloaded can be overwritten by setting the `MODELZOO_VERSION` cmake variable. + +#### Assumptions / Known limits + +If several packages make use of the same neural network, it will be downloaded once per package. + +In case a requested artifact doesn't exist in the S3 bucket, the error message from ExternalProject is not explicit enough for the user to understand what went wrong. + +In case the user manually sets `MODELZOO_VERSION` to "latest", the archive will not be re-downloaded when it gets updated in the S3 bucket (it is not a problem for tagged versions as they are not expected to be updated). + +#### Inputs / Outputs + +Inputs: + +- `DOWNLOAD_ARTIFACTS` cmake variable; needs to be set to enable downloading the artifacts +- `MODELZOO_VERSION` cmake variable; can be used to overwrite the default target version of downloads + +Outputs: + +- `get_neural_network` cmake function; can be used to get a neural network compiled for a specific backend + +In/Out: + +- The `DEPENDENCY` argument of `get_neural_network` can be checked for the outcome of the function. + It is an empty string when the neural network couldn't be made available. + ## Security considerations +### Pipeline + Both the input and output are controlled by the same actor, so the following security concerns are out-of-scope: - Spoofing @@ -61,6 +130,16 @@ run code on the CPU, which would already allow a more severe Denial-of-Service a No elevation of privilege is required for this package. +### Network provider + +The pre-compiled networks are downloaded from an S3 bucket and are under threat of spoofing, +tampering and denial of service. +Spoofing is mitigated by using an https connection. +Mitigations for tampering and denial of service are left to AWS. + +The user-provided networks are installed as they are on the host system. +The user is in charge of securing the files they provide with regard to information disclosure. + ## Future extensions / Unimplemented parts Future packages will use tvm_utility as part of the perception stack to run machine learning workloads. diff --git a/common/tvm_utility/include/tvm_utility/pipeline.hpp b/common/tvm_utility/include/tvm_utility/pipeline.hpp index 1366345560807..a6578ab885490 100644 --- a/common/tvm_utility/include/tvm_utility/pipeline.hpp +++ b/common/tvm_utility/include/tvm_utility/pipeline.hpp @@ -113,7 +113,7 @@ class InferenceEngine : public PipelineStage>; typedef struct { @@ -203,12 +203,12 @@ typedef struct class InferenceEngineTVM : public InferenceEngine { public: - explicit InferenceEngineTVM(const InferenceEngineTVMConfig & config) : config_(config) + explicit InferenceEngineTVM(const InferenceEngineTVMConfig & config, const std::string & pkg_name) + : config_(config) { // Get full network path - std::string network_prefix = - ament_index_cpp::get_package_share_directory("neural_networks_provider") + "/networks/" + - config.network_name + "/" + config.network_backend + "/"; + std::string network_prefix = ament_index_cpp::get_package_share_directory(pkg_name) + + "/models/" + config.network_name + "/"; std::string network_module_path = network_prefix + config.network_module_path; std::string network_graph_path = network_prefix + config.network_graph_path; std::string network_params_path = network_prefix + config.network_params_path; diff --git a/common/tvm_utility/model_zoo.hpp.in b/common/tvm_utility/model_zoo.hpp.in deleted file mode 100644 index 0ced1b6df35c0..0000000000000 --- a/common/tvm_utility/model_zoo.hpp.in +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2021 Arm Limited and Contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef TVM_UTILITY__MODEL_ZOO_HPP_ -#define TVM_UTILITY__MODEL_ZOO_HPP_ - -#ifndef NETWORKS_BACKEND -#define NETWORKS_BACKEND llvm -#endif // BACKEND - -@GENERATED_MODEL_INCLUDES@ - -#endif // TVM_UTILITY__MODEL_ZOO_HPP_ diff --git a/common/tvm_utility/package.xml b/common/tvm_utility/package.xml index 0de99227394ce..e648a3bd47b59 100644 --- a/common/tvm_utility/package.xml +++ b/common/tvm_utility/package.xml @@ -20,20 +20,18 @@ tvm_utility 1.0.0 - A set of utility functions to help build an machine learning pipeline using + A set of utility functions to help build a machine learning pipeline using TVM as the inference engine. Ambroise Vincent Apache License 2.0 autoware_cmake - tvm_vendor ament_index_cpp libopenblas-dev libopencv-dev - neural_networks_provider - sensor_msgs + tvm_vendor ament_lint_auto autoware_lint_common diff --git a/common/tvm_utility/test/yolo_v2_tiny/main.cpp b/common/tvm_utility/test/yolo_v2_tiny/main.cpp index de11b31d067f4..cc96bfe8e465b 100644 --- a/common/tvm_utility/test/yolo_v2_tiny/main.cpp +++ b/common/tvm_utility/test/yolo_v2_tiny/main.cpp @@ -13,8 +13,8 @@ // limitations under the License. #include "gtest/gtest.h" -#include "tvm_utility/model_zoo.hpp" #include "tvm_utility/pipeline.hpp" +#include "yolo_v2_tiny/inference_engine_tvm_config.hpp" #include @@ -240,7 +240,7 @@ TEST(PipelineExamples, SimplePipeline) using PostPT = PostProcessorYoloV2Tiny; PrePT PreP{config}; - IET IE{config}; + IET IE{config, "tvm_utility"}; PostPT PostP{config}; tvm_utility::pipeline::Pipeline pipeline(PreP, IE, PostP); diff --git a/common/tvm_utility/tvm_utility-extras.cmake b/common/tvm_utility/tvm_utility-extras.cmake new file mode 100644 index 0000000000000..f8c1583b28f8c --- /dev/null +++ b/common/tvm_utility/tvm_utility-extras.cmake @@ -0,0 +1,84 @@ +# Copyright 2021-2022 Arm Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get user-provided variables +set(DOWNLOAD_ARTIFACTS OFF CACHE BOOL "enable artifacts download") +set(MODELZOO_VERSION "1.3.0-20220902" CACHE STRING "targeted ModelZoo version") + +# +# Download the selected neural network if it is not already present on disk. +# Make inference_engine_tvm_config.hpp available under "data/models/${MODEL_NAME}/". +# Install the TVM artifacts to "share/${PROJECT_NAME}/models/". +# Return the name of the custom target in the DEPENDENCY parameter. +# +# :param MODEL_NAME: the name of the targeted neural network +# :type MODEL_NAME: string +# :param MODEL_BACKEND: the name of the targeted backend +# :type MODEL_BACKEND: string +# :param DEPENDENCY: output parameter; name of the ExternalProject top level target +# :type DEPENDENCY: string +# +function(get_neural_network MODEL_NAME MODEL_BACKEND DEPENDENCY) + set(DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}/data) + set(EXTERNALPROJECT_NAME ${MODEL_NAME}_${MODEL_BACKEND}) + + # Prioritize user-provided models. + if(IS_DIRECTORY "${DATA_PATH}/user/${MODEL_NAME}") + message(STATUS "Using user-provided model from ${DATA_PATH}/user/${MODEL_NAME}") + file(REMOVE_RECURSE "${DATA_PATH}/models/${MODEL_NAME}/") + configure_file( + "${DATA_PATH}/user/${MODEL_NAME}/inference_engine_tvm_config.hpp" + "${DATA_PATH}/models/${MODEL_NAME}/inference_engine_tvm_config.hpp" + COPYONLY + ) + install( + DIRECTORY "${DATA_PATH}/user/${MODEL_NAME}" + DESTINATION "share/${PROJECT_NAME}/models/" + USE_SOURCE_PERMISSIONS + ) + else() + set(ARCHIVE_NAME "${MODEL_NAME}-${CMAKE_SYSTEM_PROCESSOR}-${MODEL_BACKEND}-${MODELZOO_VERSION}.tar.gz") + + # Use previously-downloaded archives if available. + if(EXISTS "${DATA_PATH}/downloads/${ARCHIVE_NAME}") + set(URL "${DATA_PATH}/downloads/${ARCHIVE_NAME}") + elseif(DOWNLOAD_ARTIFACTS) + message(STATUS "Downloading ${ARCHIVE_NAME} ...") + set(URL "https://autoware-modelzoo.s3.us-east-2.amazonaws.com/models/${MODELZOO_VERSION}/${ARCHIVE_NAME}") + else() + message(WARNING "Skipped download for ${MODEL_NAME} (enable by setting DOWNLOAD_ARTIFACTS)") + set(${DEPENDENCY} "" PARENT_SCOPE) + return() + endif() + + include(ExternalProject) + externalproject_add(${EXTERNALPROJECT_NAME} + DOWNLOAD_DIR "${DATA_PATH}/downloads" + SOURCE_DIR "${DATA_PATH}/models/${MODEL_NAME}" + URL ${URL} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + BUILD_BYPRODUCTS "${DATA_PATH}/models/${MODEL_NAME}/inference_engine_tvm_config.hpp" + INSTALL_COMMAND "" + ) + install( + DIRECTORY "${DATA_PATH}/models/${MODEL_NAME}" + DESTINATION "share/${PROJECT_NAME}/models/" + USE_SOURCE_PERMISSIONS + ) + endif() + + set(${DEPENDENCY} ${EXTERNALPROJECT_NAME} PARENT_SCOPE) + +endfunction() diff --git a/common/tvm_utility/tvm_utility-extras.cmake.in b/common/tvm_utility/tvm_utility-extras.cmake.in deleted file mode 100644 index 9e10dea677f78..0000000000000 --- a/common/tvm_utility/tvm_utility-extras.cmake.in +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2021-2022 Arm Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Search through the neural networks made available by the neural_networks_provider -# package and check if the targeted network exists. -# -# :param target_network_name: the name of the targeted network -# :type target_network_name: string -# :param target_network_backend: the name of the targeted backend -# :type target_network_backend: string -# -macro(autoware_check_neural_network target_network_name target_network_backend) - - normalize_path(_target_dir "@tvm_utility_NETWORKS_DIR@/${target_network_name}/${target_network_backend}") - - if(IS_DIRECTORY ${_target_dir} - AND EXISTS "${_target_dir}/inference_engine_tvm_config.hpp" - AND EXISTS "${_target_dir}/deploy_lib.so" - AND EXISTS "${_target_dir}/deploy_graph.json" - AND EXISTS "${_target_dir}/deploy_param.params" - ) - set(${target_network_name}_FOUND TRUE) - else() - set(${target_network_name}_FOUND FALSE) - endif() - -endmacro() diff --git a/perception/lidar_apollo_segmentation_tvm/.gitignore b/perception/lidar_apollo_segmentation_tvm/.gitignore new file mode 100644 index 0000000000000..8fce603003c1e --- /dev/null +++ b/perception/lidar_apollo_segmentation_tvm/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/perception/lidar_apollo_segmentation_tvm/CMakeLists.txt b/perception/lidar_apollo_segmentation_tvm/CMakeLists.txt index f665a22a6005b..68d9e820a57d5 100644 --- a/perception/lidar_apollo_segmentation_tvm/CMakeLists.txt +++ b/perception/lidar_apollo_segmentation_tvm/CMakeLists.txt @@ -22,14 +22,20 @@ set(tvm_runtime_DIR ${tvm_vendor_DIR}) find_package(tvm_runtime CONFIG REQUIRED) find_package(PCL 1.10 REQUIRED) -# Get target backend +# Used in "extras" template file +set(${PROJECT_NAME}_BUILT FALSE) + +# Gather neural network information. set(${PROJECT_NAME}_BACKEND llvm CACHE STRING "${PROJECT_NAME} neural network backend") +set(MODEL_NAME baidu_cnn) -# Test if files exist. The result is set in baidu_cnn_FOUND -autoware_check_neural_network(baidu_cnn "${${PROJECT_NAME}_BACKEND}") -if(baidu_cnn_FOUND) +# Get neural network. +set(NN_DEPENDENCY "") +get_neural_network(${MODEL_NAME} ${${PROJECT_NAME}_BACKEND} NN_DEPENDENCY) +if(NOT NN_DEPENDENCY STREQUAL "") # Library ament_auto_add_library(${PROJECT_NAME} SHARED + data/models/${MODEL_NAME}/inference_engine_tvm_config.hpp include/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm.hpp include/lidar_apollo_segmentation_tvm/cluster2d.hpp include/lidar_apollo_segmentation_tvm/disjoint_set.hpp @@ -43,17 +49,20 @@ if(baidu_cnn_FOUND) src/feature_map.cpp src/log_table.cpp ) + add_dependencies(${PROJECT_NAME} ${NN_DEPENDENCY}) target_compile_options(${PROJECT_NAME} PRIVATE "-Wno-sign-conversion" "-Wno-conversion") - target_compile_definitions(${PROJECT_NAME} PRIVATE NETWORKS_BACKEND=${${PROJECT_NAME}_BACKEND}) - # Set includes as "SYSTEM" to avoid compiler errors on their headers. target_include_directories(${PROJECT_NAME} SYSTEM PUBLIC "${PCL_INCLUDE_DIRS}" "${tvm_vendor_INCLUDE_DIRS}" ) + target_include_directories(${PROJECT_NAME} PRIVATE + data/models + ) + target_link_libraries(${PROJECT_NAME} ${PCL_LIBRARIES} ${tf2_ros_LIBRARIES} @@ -69,8 +78,10 @@ if(baidu_cnn_FOUND) endif() ament_export_include_directories(${PCL_INCLUDE_DIRS}) - ament_auto_package() + set(${PROJECT_NAME}_BUILT TRUE) else() message(WARNING "Neural network not found, skipping package.") - ament_auto_package() endif() + +list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${PROJECT_NAME}-extras.cmake.in") +ament_auto_package() diff --git a/perception/lidar_apollo_segmentation_tvm/design/lidar-segmentation-design.md b/perception/lidar_apollo_segmentation_tvm/design/lidar-segmentation-design.md index ead0672ffe408..ac0d17a7139a8 100644 --- a/perception/lidar_apollo_segmentation_tvm/design/lidar-segmentation-design.md +++ b/perception/lidar_apollo_segmentation_tvm/design/lidar-segmentation-design.md @@ -7,7 +7,7 @@ #### Neural network This package will not build without a neural network for its inference. -The network is provided by the neural_networks_provider package. +The network is provided by the cmake function exported by the tvm_utility package. See its design page for more information on how to enable downloading pre-compiled networks (by setting the `DOWNLOAD_ARTIFACTS` cmake variable), or how to handle user-compiled networks. #### Backend @@ -24,6 +24,10 @@ This package instead relies on further processing by a dedicated shape estimator Note: the parameters described in the original design have been modified and are out of date. +### Inputs / Outputs / API + +The package exports a boolean `lidar_apollo_segmentation_tvm_BUILT` cmake variable. + ## Reference Lidar segmentation is based off a core algorithm by [Apollo](https://github.com/ApolloAuto/apollo/blob/master/docs/specs/3d_obstacle_perception.md), with modifications from [TierIV] () for the TVM backend. diff --git a/perception/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm-extras.cmake.in b/perception/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm-extras.cmake.in new file mode 100644 index 0000000000000..2ecef15eb4d26 --- /dev/null +++ b/perception/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm-extras.cmake.in @@ -0,0 +1,15 @@ +# Copyright 2022 Arm Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(lidar_apollo_segmentation_tvm_BUILT @lidar_apollo_segmentation_tvm_BUILT@) diff --git a/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp b/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp index 134f955127c30..6afa123b9317e 100644 --- a/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp +++ b/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include -#include #include #include @@ -120,7 +120,7 @@ ApolloLidarSegmentation::ApolloLidarSegmentation( pcl_pointcloud_ptr_(new pcl::PointCloud), PreP(std::make_shared( config, range, use_intensity_feature, use_constant_feature, min_height, max_height)), - IE(std::make_shared(config)), + IE(std::make_shared(config, "lidar_apollo_segmentation_tvm")), PostP(std::make_shared( config, pcl_pointcloud_ptr_, range, objectness_thresh, score_threshold, height_thresh, min_pts_num)), diff --git a/perception/lidar_apollo_segmentation_tvm_nodes/CMakeLists.txt b/perception/lidar_apollo_segmentation_tvm_nodes/CMakeLists.txt index 202cac6ad328a..f42ee8e97c08e 100644 --- a/perception/lidar_apollo_segmentation_tvm_nodes/CMakeLists.txt +++ b/perception/lidar_apollo_segmentation_tvm_nodes/CMakeLists.txt @@ -18,9 +18,8 @@ project(lidar_apollo_segmentation_tvm_nodes) find_package(autoware_cmake REQUIRED) autoware_package() -# Check that a neural network was available for lidar_apollo_segmentation_tvm to be built. -autoware_check_neural_network(baidu_cnn "${${lidar_apollo_segmentation_tvm}_BACKEND}") -if(baidu_cnn_FOUND) +# Only build if the "core" package has been built. +if(lidar_apollo_segmentation_tvm_BUILT) # Set lidar_apollo_segmentation_tvm includes as "SYSTEM" to ignore compiler errors on PCL headers include_directories(SYSTEM "${lidar_apollo_segmentation_tvm_INCLUDE_DIRS}") @@ -48,6 +47,6 @@ if(baidu_cnn_FOUND) param ) else() - message(WARNING "lidar_apollo_segmentation_tvm not found, skipping package.") + message(WARNING "lidar_apollo_segmentation_tvm not built, skipping package.") ament_auto_package() endif()