diff --git a/common/neural_networks_provider/CMakeLists.txt b/common/neural_networks_provider/CMakeLists.txt
deleted file mode 100644
index bf4e5ec03d84d..0000000000000
--- a/common/neural_networks_provider/CMakeLists.txt
+++ /dev/null
@@ -1,63 +0,0 @@
-# Copyright 2021-2022 Arm Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cmake_minimum_required(VERSION 3.14)
-project(neural_networks_provider)
-
-find_package(autoware_cmake REQUIRED)
-autoware_package()
-
-set(SUPPORTED_ARCH "aarch64" "x86_64")
-set(USER_DIR ${CMAKE_CURRENT_SOURCE_DIR}/user/${CMAKE_SYSTEM_PROCESSOR})
-
-set(PREFIX_DIR ${CMAKE_BINARY_DIR}/download)
-file(MAKE_DIRECTORY ${PREFIX_DIR}/src/networks)
-
-if(CMAKE_SYSTEM_PROCESSOR IN_LIST SUPPORTED_ARCH)
- if(DOWNLOAD_ARTIFACTS)
- set(EXEC_DOWNLOAD TRUE)
- else()
- set(EXEC_DOWNLOAD FALSE)
- message(WARNING "Skipped download (enable by setting DOWNLOAD_ARTIFACTS)")
- endif()
-else()
- set(EXEC_DOWNLOAD FALSE)
- message(WARNING "Skipped download (unsupported architecture)")
-endif()
-
-add_custom_target(artifact_download ALL
- COMMAND "EXEC_DOWNLOAD=${EXEC_DOWNLOAD}"
- "PREFIX_DIR=${PREFIX_DIR}"
- "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}"
- "${CMAKE_CURRENT_SOURCE_DIR}/download_neural_networks.sh"
-)
-
-# Install the pre-compiled networks
-install(
- DIRECTORY ${PREFIX_DIR}/src/networks
- DESTINATION share/${PROJECT_NAME}/
- USE_SOURCE_PERMISSIONS
-)
-
-# Install user-provided networks over the pre-compiled ones
-if(EXISTS ${USER_DIR})
- install(
- DIRECTORY ${USER_DIR}/
- DESTINATION share/${PROJECT_NAME}/networks/
- USE_SOURCE_PERMISSIONS
- )
-endif()
-
-list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "neural_networks_provider-extras.cmake")
-ament_auto_package()
diff --git a/common/neural_networks_provider/design/neural_networks-design.md b/common/neural_networks_provider/design/neural_networks-design.md
deleted file mode 100644
index 8532298f21271..0000000000000
--- a/common/neural_networks_provider/design/neural_networks-design.md
+++ /dev/null
@@ -1,101 +0,0 @@
-# Neural Networks Provider design {#neural-networks-provider-design}
-
-## Purpose / Use cases
-
-
-
-
-This package provides pre-compiled neural networks to packages using them for their inference.
-
-## Design
-
-
-
-
-The neural networks are compiled as part of the
-[Model Zoo](https://github.com/autowarefoundation/modelzoo/) CI pipeline and saved to an S3 bucket.
-They are downloaded and installed when this package is built when the `DOWNLOAD_ARTIFACTS` variable
-is set.
-
-The user can provide its own compiled networks to be made available through this package, or to
-overwrite the pre-compiled ones by creating a `user` directory and using the same directory
-structure.
-
-The structure is as follow:
-
-```{text}
-.
-├── ${ARCH 1}
-│ ├── ${MODEL 1}
-│ │ ├── ${BACKEND 1}
-│ │ │ ├── deploy_graph.json
-│ │ │ ├── deploy_lib.so
-│ │ │ ├── deploy_param.params
-│ │ │ └── inference)_engine_tvm_config.hpp
-│ │ └── ${BACKEND ...}
-│ │ └── ...
-│ └── ${MODEL ...}
-│ └── ...
-└── ${ARCH ...}
- └── ...
-```
-
-The pre-compiled networks are downloaded at build time when a new revision is available, provided
-that `DOWNLOAD_ARTIFACTS` is set (see Inputs).
-
-### Assumptions / Known limits
-
-
-
-An internet connection is required at build time.
-
-### Inputs / Outputs / API
-
-
-
-
-Inputs:
-
-- `DOWNLOAD_ARTIFACTS` variable needs to be set to enable downloading the artifacts
-
-Outputs:
-
-- `neural_networks_provider_NETWORKS_DIR` variable containing the path to the root directory of the networks
-- `neural_networks_provider_NAMES` variable containing the list of available networks
-
-API: none
-
-### Error detection and handling
-
-
-
-Building the package will not fail.
-If no models are available for the target architecture, then a log message is displayed.
-
-## Security considerations
-
-
-
-
-The pre-compiled networks are downloaded from an S3 bucket and are under threat of spoofing,
-tampering and denial of service.
-Spoofing is mitigated by using an https connection.
-Mitigations for tampering and denial of service are left to AWS.
-
-The user-provided networks are installed as they are on the host system.
-The user is in charge of securing the files they provide with regard to information disclosure.
-
-## Related issues
-
-
-
-- #226: Autoware.Auto Neural Networks Inference Architecture Design
diff --git a/common/neural_networks_provider/download_neural_networks.sh b/common/neural_networks_provider/download_neural_networks.sh
deleted file mode 100755
index 098da63e6f7ff..0000000000000
--- a/common/neural_networks_provider/download_neural_networks.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh -eu
-
-# Copyright (c) 2021-2022, Arm Limited
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The variables "EXEC_DOWNLOAD", "PREFIX_DIR" and "CMAKE_SYSTEM_PROCESSOR" are
-# passed to this script by the CmakeLists.txt of the package itself.
-
-NETWORKS_FILE="$PREFIX_DIR"/src/networks-"$CMAKE_SYSTEM_PROCESSOR".tar.gz
-S3_BUCKET=https://autoware-modelzoo.s3.us-east-2.amazonaws.com/
-
-# Function to download-extract the networks artifact
-download_artifact() {
- curl -sS -o "$NETWORKS_FILE" "$S3_BUCKET"networks-"$CMAKE_SYSTEM_PROCESSOR".tar.gz
- tar -zxvf "$NETWORKS_FILE" -C "$PREFIX_DIR"/src/networks
-}
-
-if [ "$EXEC_DOWNLOAD" = "TRUE" ]; then
-
- # If the networks artifact is not present, simply download
- if [ ! -f "$NETWORKS_FILE" ]; then
- download_artifact
-
- # If the networks artifact is already present, check the MD5 hash and, if needed, re-download it
- else
- OLD_HASH=$(md5sum "$NETWORKS_FILE" | cut -d ' ' -f 1)
- NEW_HASH=$(curl -sS "$S3_BUCKET"networks-"$CMAKE_SYSTEM_PROCESSOR".md5)
- if [ "$OLD_HASH" != "$NEW_HASH" ]; then
- download_artifact
- fi
- fi
-fi
diff --git a/common/neural_networks_provider/neural_networks_provider-extras.cmake b/common/neural_networks_provider/neural_networks_provider-extras.cmake
deleted file mode 100644
index 2a1168f8674ce..0000000000000
--- a/common/neural_networks_provider/neural_networks_provider-extras.cmake
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright 2021-2022 Arm Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-normalize_path(neural_networks_provider_NETWORKS_DIR
- "${neural_networks_provider_DIR}/../networks"
-)
-if(NOT IS_DIRECTORY "${neural_networks_provider_NETWORKS_DIR}")
- message(WARNING "Package 'neural_networks_provider' exports the directory '${neural_networks_provider_NETWORKS_DIR}' which doesn't exist")
-endif()
-
-macro(_subdirlist result dir)
- file(GLOB children RELATIVE ${dir} ${dir}/*)
- set(${result} "")
- foreach(child ${children})
- if(IS_DIRECTORY ${dir}/${child})
- list(APPEND ${result} ${child})
- endif()
- endforeach()
-endmacro()
-
-_subdirlist(neural_networks_provider_NAMES ${neural_networks_provider_NETWORKS_DIR})
diff --git a/common/neural_networks_provider/package.xml b/common/neural_networks_provider/package.xml
deleted file mode 100644
index 678fc8ebc2531..0000000000000
--- a/common/neural_networks_provider/package.xml
+++ /dev/null
@@ -1,23 +0,0 @@
-
-
-
- neural_networks_provider
- 1.0.0
- A wrapper package providing compiled neural networks.
- Ambroise Vincent
- Apache 2.0
-
- ament_cmake_auto
- coreutils
- curl
- tar
-
- autoware_cmake
-
- ament_lint_auto
- autoware_lint_common
-
-
- ament_cmake
-
-
diff --git a/common/tvm_utility/.gitignore b/common/tvm_utility/.gitignore
index ade3ffe59a47f..a09bb7234b379 100644
--- a/common/tvm_utility/.gitignore
+++ b/common/tvm_utility/.gitignore
@@ -1,3 +1,2 @@
-libs
-*.o
-*.so
+artifacts/**/*.jpg
+data/
diff --git a/common/tvm_utility/CMakeLists.txt b/common/tvm_utility/CMakeLists.txt
index e4be23f40fca7..b49f141d9e2e4 100644
--- a/common/tvm_utility/CMakeLists.txt
+++ b/common/tvm_utility/CMakeLists.txt
@@ -15,37 +15,20 @@
cmake_minimum_required(VERSION 3.14)
project(tvm_utility)
+include("${PROJECT_NAME}-extras.cmake")
+
find_package(autoware_cmake REQUIRED)
autoware_package()
-# Configure
-
-set(tvm_utility_NETWORKS_DIR ${neural_networks_provider_NETWORKS_DIR})
-configure_file(tvm_utility-extras.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake @ONLY)
-
-foreach(network_name ${neural_networks_provider_NAMES})
- list(APPEND MODEL_INCLUDES_LIST
- "#define INCLUDE <${network_name}/NETWORKS_BACKEND/inference_engine_tvm_config.hpp>"
- "#include INCLUDE"
- "#undef INCLUDE"
- )
-endforeach()
-list(JOIN MODEL_INCLUDES_LIST "\n" GENERATED_MODEL_INCLUDES)
-configure_file(model_zoo.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/tvm_utility/model_zoo.hpp)
-
# Library
set(TVM_UTILITY_NODE_LIB_HEADERS
"include/${PROJECT_NAME}/pipeline.hpp"
- "${CMAKE_CURRENT_BINARY_DIR}/include/tvm_utility/model_zoo.hpp"
)
ament_auto_add_library(${PROJECT_NAME} SHARED ${TVM_UTILITY_NODE_LIB_HEADERS})
set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
-ament_export_include_directories(${tvm_utility_NETWORKS_DIR})
-install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/ DESTINATION include)
-
if(BUILD_TESTING)
# Get target backend
set(${PROJECT_NAME}_BACKEND llvm CACHE STRING "${PROJECT_NAME} neural network backend")
@@ -53,10 +36,8 @@ if(BUILD_TESTING)
# compile each folder inside test/ as a test case
find_package(ament_cmake_gtest REQUIRED)
find_package(OpenCV REQUIRED)
- find_package(tvm_vendor REQUIRED)
set(tvm_runtime_DIR ${tvm_vendor_DIR})
find_package(tvm_runtime CONFIG REQUIRED)
- include(${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake)
set(TEST_ARTIFACTS "${CMAKE_CURRENT_LIST_DIR}/artifacts")
file(GLOB TEST_CASES test/*)
@@ -67,10 +48,11 @@ if(BUILD_TESTING)
# the folder name becomes the test case name
file(RELATIVE_PATH TEST_CASE_NAME ${CMAKE_CURRENT_LIST_DIR}/test ${TEST_FOLDER})
- # Test if files exist. The result is set in ${TEST_CASE_NAME}_FOUND
- autoware_check_neural_network(${TEST_CASE_NAME} "${${PROJECT_NAME}_BACKEND}")
+ # Get neural network.
+ set(NN_DEPENDENCY "")
+ get_neural_network(${TEST_CASE_NAME} ${${PROJECT_NAME}_BACKEND} NN_DEPENDENCY)
- if(${TEST_CASE_NAME}_FOUND)
+ if(NOT NN_DEPENDENCY STREQUAL "")
if(TEST_CASE_NAME STREQUAL "yolo_v2_tiny" AND
NOT EXISTS ${TEST_ARTIFACTS}/yolo_v2_tiny/test_image_0.jpg)
message(WARNING "Missing image artifact for yolo_v2_tiny, skipping test")
@@ -82,7 +64,8 @@ if(BUILD_TESTING)
ament_target_dependencies(${TEST_CASE_NAME}
"ament_index_cpp"
"tvm_vendor"
- "sensor_msgs")
+ )
+ add_dependencies(${TEST_CASE_NAME} ${NN_DEPENDENCY})
target_link_libraries("${TEST_CASE_NAME}"
"${OpenCV_LIBRARIES}"
@@ -91,14 +74,11 @@ if(BUILD_TESTING)
target_include_directories("${TEST_CASE_NAME}" SYSTEM PUBLIC
"${OpenCV_INCLUDE_DIRS}"
- "${tvm_vendor_INCLUDE_DIRS}"
+ "${tvm_utility_FOUND_INCLUDE_DIRS}"
+ "data/models"
"include"
- "${CMAKE_CURRENT_BINARY_DIR}/include"
- "${tvm_utility_NETWORKS_DIR}"
)
- target_compile_definitions(${TEST_CASE_NAME} PRIVATE NETWORKS_BACKEND=${${PROJECT_NAME}_BACKEND})
-
# Install test-specific files
if(IS_DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME})
install(DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME}/
@@ -113,5 +93,5 @@ if(BUILD_TESTING)
endforeach()
endif()
-list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake")
+list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${PROJECT_NAME}-extras.cmake")
ament_auto_package()
diff --git a/common/tvm_utility/design/tvm-utility-design.md b/common/tvm_utility/design/tvm-utility-design.md
index 978bc0c2576f9..2e322261152c0 100644
--- a/common/tvm_utility/design/tvm-utility-design.md
+++ b/common/tvm_utility/design/tvm-utility-design.md
@@ -14,7 +14,7 @@ The Pipeline Class is a standardized way to write an inference pipeline. The pip
the pre-processor, the inference engine and the post-processor. The TVM implementation of an inference engine stage is
provided.
-### Inputs / Outputs / API
+### API
The pre-processor and post-processor need to be implemented by the user before instantiating the pipeline. You can see example
usage in this [example_pipeline](../test/yolo_v2_tiny).
@@ -30,23 +30,92 @@ int main() {
}
```
-#### Outputs
+#### Models
-- `autoware_check_neural_network` cmake macro to check if a specific network and backend combination exists
-
-#### Backend
-
-Dependent packages are expected to include `model_zoo.hpp` in order to get the TVM configuration structure of the targeted model/backend combination.
-The backend used to do the inference can be specified by setting `NETWORKS_BACKEND` as a compile definition.
-It defaults to `llvm`.
+Dependent packages are expected to use the `get_neural_network` cmake function from this package in order to get the compiled TVM models.
### Error detection and handling
`std::runtime_error` should be thrown whenever an error is encountered. It should be populated with an appropriate text
error description.
+### Neural Networks Provider
+
+This package also provides a utility to get pre-compiled neural networks to packages using them for their inference.
+
+The neural networks are compiled as part of the
+[Model Zoo](https://github.com/autowarefoundation/modelzoo/) CI pipeline and saved to an S3 bucket.
+This package exports cmake variables and functions for ease of access to those neural networks.
+
+The `get_neural_network` function creates an abstraction for the artifact management.
+The artifacts are saved under the source directory of the package making use of the function; under "data/".
+Priority is given to user-provided files, under "data/user/${MODEL_NAME}/".
+If there are no user-provided files, the function tries to reuse previously-downloaded artifacts.
+If there are no previously-downloaded artifacts, and if the `DOWNLOAD_ARTIFACTS` cmake variable is set, they will be downloaded from the bucket.
+Otherwise, nothing happens.
+
+The structure inside of the source directory of the package making use of the function is as follow:
+
+```{text}
+.
+├── data
+│ ├── downloads
+│ │ ├── ${MODEL 1}-${ARCH 1}-{BACKEND 1}-{VERSION 1}.tar.gz
+│ │ ├── ...
+│ │ └── ${MODEL ...}-${ARCH ...}-{BACKEND ...}-{VERSION ...}.tar.gz
+│ ├── models
+│ │ ├── ${MODEL 1}
+│ │ │ ├── ...
+│ │ │ └── inference_engine_tvm_config.hpp
+│ │ ├── ...
+│ │ └── ${MODEL ...}
+│ │ └── ...
+│ └── user
+│ ├── ${MODEL 1}
+│ │ ├── deploy_graph.json
+│ │ ├── deploy_lib.so
+│ │ ├── deploy_param.params
+│ │ └── inference_engine_tvm_config.hpp
+│ ├── ...
+│ └── ${MODEL ...}
+│ └── ...
+```
+
+The `inference_engine_tvm_config.hpp` file needed for compilation by dependent packages is made available under "data/models/${MODEL_NAME}/inference_engine_tvm_config.hpp".
+Dependent packages can use the cmake `add_dependencies` function with the name provided in the `DEPENDENCY` output parameter of `get_neural_network` to ensure this file is created before it gets used.
+
+The other `deploy_*` files are installed to "models/${MODEL_NAME}/" under the `share` directory of the package.
+
+The target version to be downloaded can be overwritten by setting the `MODELZOO_VERSION` cmake variable.
+
+#### Assumptions / Known limits
+
+If several packages make use of the same neural network, it will be downloaded once per package.
+
+In case a requested artifact doesn't exist in the S3 bucket, the error message from ExternalProject is not explicit enough for the user to understand what went wrong.
+
+In case the user manually sets `MODELZOO_VERSION` to "latest", the archive will not be re-downloaded when it gets updated in the S3 bucket (it is not a problem for tagged versions as they are not expected to be updated).
+
+#### Inputs / Outputs
+
+Inputs:
+
+- `DOWNLOAD_ARTIFACTS` cmake variable; needs to be set to enable downloading the artifacts
+- `MODELZOO_VERSION` cmake variable; can be used to overwrite the default target version of downloads
+
+Outputs:
+
+- `get_neural_network` cmake function; can be used to get a neural network compiled for a specific backend
+
+In/Out:
+
+- The `DEPENDENCY` argument of `get_neural_network` can be checked for the outcome of the function.
+ It is an empty string when the neural network couldn't be made available.
+
## Security considerations
+### Pipeline
+
Both the input and output are controlled by the same actor, so the following security concerns are out-of-scope:
- Spoofing
@@ -61,6 +130,16 @@ run code on the CPU, which would already allow a more severe Denial-of-Service a
No elevation of privilege is required for this package.
+### Network provider
+
+The pre-compiled networks are downloaded from an S3 bucket and are under threat of spoofing,
+tampering and denial of service.
+Spoofing is mitigated by using an https connection.
+Mitigations for tampering and denial of service are left to AWS.
+
+The user-provided networks are installed as they are on the host system.
+The user is in charge of securing the files they provide with regard to information disclosure.
+
## Future extensions / Unimplemented parts
Future packages will use tvm_utility as part of the perception stack to run machine learning workloads.
diff --git a/common/tvm_utility/include/tvm_utility/pipeline.hpp b/common/tvm_utility/include/tvm_utility/pipeline.hpp
index 1366345560807..a6578ab885490 100644
--- a/common/tvm_utility/include/tvm_utility/pipeline.hpp
+++ b/common/tvm_utility/include/tvm_utility/pipeline.hpp
@@ -113,7 +113,7 @@ class InferenceEngine : public PipelineStage>;
typedef struct
{
@@ -203,12 +203,12 @@ typedef struct
class InferenceEngineTVM : public InferenceEngine
{
public:
- explicit InferenceEngineTVM(const InferenceEngineTVMConfig & config) : config_(config)
+ explicit InferenceEngineTVM(const InferenceEngineTVMConfig & config, const std::string & pkg_name)
+ : config_(config)
{
// Get full network path
- std::string network_prefix =
- ament_index_cpp::get_package_share_directory("neural_networks_provider") + "/networks/" +
- config.network_name + "/" + config.network_backend + "/";
+ std::string network_prefix = ament_index_cpp::get_package_share_directory(pkg_name) +
+ "/models/" + config.network_name + "/";
std::string network_module_path = network_prefix + config.network_module_path;
std::string network_graph_path = network_prefix + config.network_graph_path;
std::string network_params_path = network_prefix + config.network_params_path;
diff --git a/common/tvm_utility/model_zoo.hpp.in b/common/tvm_utility/model_zoo.hpp.in
deleted file mode 100644
index 0ced1b6df35c0..0000000000000
--- a/common/tvm_utility/model_zoo.hpp.in
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2021 Arm Limited and Contributors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef TVM_UTILITY__MODEL_ZOO_HPP_
-#define TVM_UTILITY__MODEL_ZOO_HPP_
-
-#ifndef NETWORKS_BACKEND
-#define NETWORKS_BACKEND llvm
-#endif // BACKEND
-
-@GENERATED_MODEL_INCLUDES@
-
-#endif // TVM_UTILITY__MODEL_ZOO_HPP_
diff --git a/common/tvm_utility/package.xml b/common/tvm_utility/package.xml
index 0de99227394ce..e648a3bd47b59 100644
--- a/common/tvm_utility/package.xml
+++ b/common/tvm_utility/package.xml
@@ -20,20 +20,18 @@
tvm_utility
1.0.0
- A set of utility functions to help build an machine learning pipeline using
+ A set of utility functions to help build a machine learning pipeline using
TVM as the inference engine.
Ambroise Vincent
Apache License 2.0
autoware_cmake
- tvm_vendor
ament_index_cpp
libopenblas-dev
libopencv-dev
- neural_networks_provider
- sensor_msgs
+ tvm_vendor
ament_lint_auto
autoware_lint_common
diff --git a/common/tvm_utility/test/yolo_v2_tiny/main.cpp b/common/tvm_utility/test/yolo_v2_tiny/main.cpp
index de11b31d067f4..cc96bfe8e465b 100644
--- a/common/tvm_utility/test/yolo_v2_tiny/main.cpp
+++ b/common/tvm_utility/test/yolo_v2_tiny/main.cpp
@@ -13,8 +13,8 @@
// limitations under the License.
#include "gtest/gtest.h"
-#include "tvm_utility/model_zoo.hpp"
#include "tvm_utility/pipeline.hpp"
+#include "yolo_v2_tiny/inference_engine_tvm_config.hpp"
#include
@@ -240,7 +240,7 @@ TEST(PipelineExamples, SimplePipeline)
using PostPT = PostProcessorYoloV2Tiny;
PrePT PreP{config};
- IET IE{config};
+ IET IE{config, "tvm_utility"};
PostPT PostP{config};
tvm_utility::pipeline::Pipeline pipeline(PreP, IE, PostP);
diff --git a/common/tvm_utility/tvm_utility-extras.cmake b/common/tvm_utility/tvm_utility-extras.cmake
new file mode 100644
index 0000000000000..f8c1583b28f8c
--- /dev/null
+++ b/common/tvm_utility/tvm_utility-extras.cmake
@@ -0,0 +1,84 @@
+# Copyright 2021-2022 Arm Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Get user-provided variables
+set(DOWNLOAD_ARTIFACTS OFF CACHE BOOL "enable artifacts download")
+set(MODELZOO_VERSION "1.3.0-20220902" CACHE STRING "targeted ModelZoo version")
+
+#
+# Download the selected neural network if it is not already present on disk.
+# Make inference_engine_tvm_config.hpp available under "data/models/${MODEL_NAME}/".
+# Install the TVM artifacts to "share/${PROJECT_NAME}/models/".
+# Return the name of the custom target in the DEPENDENCY parameter.
+#
+# :param MODEL_NAME: the name of the targeted neural network
+# :type MODEL_NAME: string
+# :param MODEL_BACKEND: the name of the targeted backend
+# :type MODEL_BACKEND: string
+# :param DEPENDENCY: output parameter; name of the ExternalProject top level target
+# :type DEPENDENCY: string
+#
+function(get_neural_network MODEL_NAME MODEL_BACKEND DEPENDENCY)
+ set(DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}/data)
+ set(EXTERNALPROJECT_NAME ${MODEL_NAME}_${MODEL_BACKEND})
+
+ # Prioritize user-provided models.
+ if(IS_DIRECTORY "${DATA_PATH}/user/${MODEL_NAME}")
+ message(STATUS "Using user-provided model from ${DATA_PATH}/user/${MODEL_NAME}")
+ file(REMOVE_RECURSE "${DATA_PATH}/models/${MODEL_NAME}/")
+ configure_file(
+ "${DATA_PATH}/user/${MODEL_NAME}/inference_engine_tvm_config.hpp"
+ "${DATA_PATH}/models/${MODEL_NAME}/inference_engine_tvm_config.hpp"
+ COPYONLY
+ )
+ install(
+ DIRECTORY "${DATA_PATH}/user/${MODEL_NAME}"
+ DESTINATION "share/${PROJECT_NAME}/models/"
+ USE_SOURCE_PERMISSIONS
+ )
+ else()
+ set(ARCHIVE_NAME "${MODEL_NAME}-${CMAKE_SYSTEM_PROCESSOR}-${MODEL_BACKEND}-${MODELZOO_VERSION}.tar.gz")
+
+ # Use previously-downloaded archives if available.
+ if(EXISTS "${DATA_PATH}/downloads/${ARCHIVE_NAME}")
+ set(URL "${DATA_PATH}/downloads/${ARCHIVE_NAME}")
+ elseif(DOWNLOAD_ARTIFACTS)
+ message(STATUS "Downloading ${ARCHIVE_NAME} ...")
+ set(URL "https://autoware-modelzoo.s3.us-east-2.amazonaws.com/models/${MODELZOO_VERSION}/${ARCHIVE_NAME}")
+ else()
+ message(WARNING "Skipped download for ${MODEL_NAME} (enable by setting DOWNLOAD_ARTIFACTS)")
+ set(${DEPENDENCY} "" PARENT_SCOPE)
+ return()
+ endif()
+
+ include(ExternalProject)
+ externalproject_add(${EXTERNALPROJECT_NAME}
+ DOWNLOAD_DIR "${DATA_PATH}/downloads"
+ SOURCE_DIR "${DATA_PATH}/models/${MODEL_NAME}"
+ URL ${URL}
+ CONFIGURE_COMMAND ""
+ BUILD_COMMAND ""
+ BUILD_BYPRODUCTS "${DATA_PATH}/models/${MODEL_NAME}/inference_engine_tvm_config.hpp"
+ INSTALL_COMMAND ""
+ )
+ install(
+ DIRECTORY "${DATA_PATH}/models/${MODEL_NAME}"
+ DESTINATION "share/${PROJECT_NAME}/models/"
+ USE_SOURCE_PERMISSIONS
+ )
+ endif()
+
+ set(${DEPENDENCY} ${EXTERNALPROJECT_NAME} PARENT_SCOPE)
+
+endfunction()
diff --git a/common/tvm_utility/tvm_utility-extras.cmake.in b/common/tvm_utility/tvm_utility-extras.cmake.in
deleted file mode 100644
index 9e10dea677f78..0000000000000
--- a/common/tvm_utility/tvm_utility-extras.cmake.in
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright 2021-2022 Arm Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Search through the neural networks made available by the neural_networks_provider
-# package and check if the targeted network exists.
-#
-# :param target_network_name: the name of the targeted network
-# :type target_network_name: string
-# :param target_network_backend: the name of the targeted backend
-# :type target_network_backend: string
-#
-macro(autoware_check_neural_network target_network_name target_network_backend)
-
- normalize_path(_target_dir "@tvm_utility_NETWORKS_DIR@/${target_network_name}/${target_network_backend}")
-
- if(IS_DIRECTORY ${_target_dir}
- AND EXISTS "${_target_dir}/inference_engine_tvm_config.hpp"
- AND EXISTS "${_target_dir}/deploy_lib.so"
- AND EXISTS "${_target_dir}/deploy_graph.json"
- AND EXISTS "${_target_dir}/deploy_param.params"
- )
- set(${target_network_name}_FOUND TRUE)
- else()
- set(${target_network_name}_FOUND FALSE)
- endif()
-
-endmacro()
diff --git a/perception/lidar_apollo_segmentation_tvm/.gitignore b/perception/lidar_apollo_segmentation_tvm/.gitignore
new file mode 100644
index 0000000000000..8fce603003c1e
--- /dev/null
+++ b/perception/lidar_apollo_segmentation_tvm/.gitignore
@@ -0,0 +1 @@
+data/
diff --git a/perception/lidar_apollo_segmentation_tvm/CMakeLists.txt b/perception/lidar_apollo_segmentation_tvm/CMakeLists.txt
index f665a22a6005b..68d9e820a57d5 100644
--- a/perception/lidar_apollo_segmentation_tvm/CMakeLists.txt
+++ b/perception/lidar_apollo_segmentation_tvm/CMakeLists.txt
@@ -22,14 +22,20 @@ set(tvm_runtime_DIR ${tvm_vendor_DIR})
find_package(tvm_runtime CONFIG REQUIRED)
find_package(PCL 1.10 REQUIRED)
-# Get target backend
+# Used in "extras" template file
+set(${PROJECT_NAME}_BUILT FALSE)
+
+# Gather neural network information.
set(${PROJECT_NAME}_BACKEND llvm CACHE STRING "${PROJECT_NAME} neural network backend")
+set(MODEL_NAME baidu_cnn)
-# Test if files exist. The result is set in baidu_cnn_FOUND
-autoware_check_neural_network(baidu_cnn "${${PROJECT_NAME}_BACKEND}")
-if(baidu_cnn_FOUND)
+# Get neural network.
+set(NN_DEPENDENCY "")
+get_neural_network(${MODEL_NAME} ${${PROJECT_NAME}_BACKEND} NN_DEPENDENCY)
+if(NOT NN_DEPENDENCY STREQUAL "")
# Library
ament_auto_add_library(${PROJECT_NAME} SHARED
+ data/models/${MODEL_NAME}/inference_engine_tvm_config.hpp
include/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm.hpp
include/lidar_apollo_segmentation_tvm/cluster2d.hpp
include/lidar_apollo_segmentation_tvm/disjoint_set.hpp
@@ -43,17 +49,20 @@ if(baidu_cnn_FOUND)
src/feature_map.cpp
src/log_table.cpp
)
+ add_dependencies(${PROJECT_NAME} ${NN_DEPENDENCY})
target_compile_options(${PROJECT_NAME} PRIVATE "-Wno-sign-conversion" "-Wno-conversion")
- target_compile_definitions(${PROJECT_NAME} PRIVATE NETWORKS_BACKEND=${${PROJECT_NAME}_BACKEND})
-
# Set includes as "SYSTEM" to avoid compiler errors on their headers.
target_include_directories(${PROJECT_NAME} SYSTEM PUBLIC
"${PCL_INCLUDE_DIRS}"
"${tvm_vendor_INCLUDE_DIRS}"
)
+ target_include_directories(${PROJECT_NAME} PRIVATE
+ data/models
+ )
+
target_link_libraries(${PROJECT_NAME}
${PCL_LIBRARIES}
${tf2_ros_LIBRARIES}
@@ -69,8 +78,10 @@ if(baidu_cnn_FOUND)
endif()
ament_export_include_directories(${PCL_INCLUDE_DIRS})
- ament_auto_package()
+ set(${PROJECT_NAME}_BUILT TRUE)
else()
message(WARNING "Neural network not found, skipping package.")
- ament_auto_package()
endif()
+
+list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${PROJECT_NAME}-extras.cmake.in")
+ament_auto_package()
diff --git a/perception/lidar_apollo_segmentation_tvm/design/lidar-segmentation-design.md b/perception/lidar_apollo_segmentation_tvm/design/lidar-segmentation-design.md
index ead0672ffe408..ac0d17a7139a8 100644
--- a/perception/lidar_apollo_segmentation_tvm/design/lidar-segmentation-design.md
+++ b/perception/lidar_apollo_segmentation_tvm/design/lidar-segmentation-design.md
@@ -7,7 +7,7 @@
#### Neural network
This package will not build without a neural network for its inference.
-The network is provided by the neural_networks_provider package.
+The network is provided by the cmake function exported by the tvm_utility package.
See its design page for more information on how to enable downloading pre-compiled networks (by setting the `DOWNLOAD_ARTIFACTS` cmake variable), or how to handle user-compiled networks.
#### Backend
@@ -24,6 +24,10 @@ This package instead relies on further processing by a dedicated shape estimator
Note: the parameters described in the original design have been modified and are out of date.
+### Inputs / Outputs / API
+
+The package exports a boolean `lidar_apollo_segmentation_tvm_BUILT` cmake variable.
+
## Reference
Lidar segmentation is based off a core algorithm by [Apollo](https://github.com/ApolloAuto/apollo/blob/master/docs/specs/3d_obstacle_perception.md), with modifications from [TierIV] () for the TVM backend.
diff --git a/perception/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm-extras.cmake.in b/perception/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm-extras.cmake.in
new file mode 100644
index 0000000000000..2ecef15eb4d26
--- /dev/null
+++ b/perception/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm-extras.cmake.in
@@ -0,0 +1,15 @@
+# Copyright 2022 Arm Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(lidar_apollo_segmentation_tvm_BUILT @lidar_apollo_segmentation_tvm_BUILT@)
diff --git a/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp b/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp
index 134f955127c30..6afa123b9317e 100644
--- a/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp
+++ b/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include
#include
#include
#include
-#include
#include
#include
@@ -120,7 +120,7 @@ ApolloLidarSegmentation::ApolloLidarSegmentation(
pcl_pointcloud_ptr_(new pcl::PointCloud),
PreP(std::make_shared(
config, range, use_intensity_feature, use_constant_feature, min_height, max_height)),
- IE(std::make_shared(config)),
+ IE(std::make_shared(config, "lidar_apollo_segmentation_tvm")),
PostP(std::make_shared(
config, pcl_pointcloud_ptr_, range, objectness_thresh, score_threshold, height_thresh,
min_pts_num)),
diff --git a/perception/lidar_apollo_segmentation_tvm_nodes/CMakeLists.txt b/perception/lidar_apollo_segmentation_tvm_nodes/CMakeLists.txt
index 202cac6ad328a..f42ee8e97c08e 100644
--- a/perception/lidar_apollo_segmentation_tvm_nodes/CMakeLists.txt
+++ b/perception/lidar_apollo_segmentation_tvm_nodes/CMakeLists.txt
@@ -18,9 +18,8 @@ project(lidar_apollo_segmentation_tvm_nodes)
find_package(autoware_cmake REQUIRED)
autoware_package()
-# Check that a neural network was available for lidar_apollo_segmentation_tvm to be built.
-autoware_check_neural_network(baidu_cnn "${${lidar_apollo_segmentation_tvm}_BACKEND}")
-if(baidu_cnn_FOUND)
+# Only build if the "core" package has been built.
+if(lidar_apollo_segmentation_tvm_BUILT)
# Set lidar_apollo_segmentation_tvm includes as "SYSTEM" to ignore compiler errors on PCL headers
include_directories(SYSTEM "${lidar_apollo_segmentation_tvm_INCLUDE_DIRS}")
@@ -48,6 +47,6 @@ if(baidu_cnn_FOUND)
param
)
else()
- message(WARNING "lidar_apollo_segmentation_tvm not found, skipping package.")
+ message(WARNING "lidar_apollo_segmentation_tvm not built, skipping package.")
ament_auto_package()
endif()