diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cae295a --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +tensorflow_cc/build/ diff --git a/Dockerfiles/install-archlinux.sh b/Dockerfiles/install-archlinux.sh index 0c35895..9384114 100755 --- a/Dockerfiles/install-archlinux.sh +++ b/Dockerfiles/install-archlinux.sh @@ -15,6 +15,7 @@ done # install requirements pacman -Syu --noconfirm --needed \ base-devel \ + gcc11 \ cmake \ git \ python \ diff --git a/Dockerfiles/install-common.sh b/Dockerfiles/install-common.sh index 1cbf91a..56d7950 100755 --- a/Dockerfiles/install-common.sh +++ b/Dockerfiles/install-common.sh @@ -19,7 +19,7 @@ chmod go+rX tensorflow_cc/tensorflow_cc/build cd tensorflow_cc/tensorflow_cc/build # build and install -cmake -DLOCAL_RAM_RESOURCES=2048 -DLOCAL_CPU_RESOURCES=2 .. +cmake -DLOCAL_RAM_RESOURCES=2048 -DLOCAL_CPU_RESOURCES=2 -DINSTALL_PROTOBUF=ON .. make rm -rf /home/tensorflow_cc/.cache rm -rf /root/.cache diff --git a/README.md b/README.md index e033de5..d5a19f0 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,12 @@ sudo pacman -S cuda cudnn nvidia ``` **Warning:** Newer versions of TensorFlow sometimes fail to build with the latest version of Bazel. You may wish -to install an older version of Bazel (e.g., 3.7.2). +to install an older version of Bazel (e.g., 5.1.1). + +**Warning:** If your program uses Protobuf and you encounter linkage or other problems, you can +try `-DINSTALL_PROTOBUF=ON` option to install a Protobuf version matching the version bundled with TensorFlow. +Our Dockerfiles are already built with the right version of Protobuf, so you may want to try +your program in the Dockerfile first. #### 2) Clone this repository ``` diff --git a/tensorflow_cc/CMakeLists.txt b/tensorflow_cc/CMakeLists.txt index 1487ecc..9e23a37 100644 --- a/tensorflow_cc/CMakeLists.txt +++ b/tensorflow_cc/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.3 FATAL_ERROR) +cmake_minimum_required(VERSION 3.7 FATAL_ERROR) cmake_policy(SET CMP0048 NEW) # Enable version parameter in project(). file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/PROJECT_VERSION" version) project( @@ -11,6 +11,7 @@ option(REQUIRE_CUDA "Make sure to find and use CUDA (implies ALLOW_CUDA)." OFF) set(LOCAL_RAM_RESOURCES 4096 CACHE STRING "The amount of local RAM resources passed to bazel scheduler (e.g., 4096).") set(LOCAL_CPU_RESOURCES HOST_CPUS CACHE STRING "The amount of local CPU cores passed to bazel scheduler (e.g., 2).") set(TENSORFLOW_TAG "v${version}" CACHE STRING "The tensorflow release tag to be checked out (default v${version}).") +option(INSTALL_PROTOBUF "Install protobuf compatible with tensorflow version." OFF) set(CMAKE_CXX_STANDARD 14 CACHE STRING "The C++ standard for building and linking the library (e.g., 14).") # ------------- @@ -31,6 +32,9 @@ configure_file("cmake/build_tensorflow.sh.in" "build_tensorflow.sh" @ONLY) # ---------------------------------------------- include(TensorflowBase) +if(INSTALL_PROTOBUF) + include(ProtobufExternal) +endif() # ------------------------------ # Define Tensorflow_CC Interface @@ -39,6 +43,10 @@ include(TensorflowBase) add_library(tensorflow_cc INTERFACE) target_compile_features(tensorflow_cc INTERFACE "cxx_std_${CMAKE_CXX_STANDARD}") +if(INSTALL_PROTOBUF) + add_dependencies(tensorflow_cc protobuf-external) +endif() + # The include folders are sometimes contained under bazel-bin/bin/ and sometimes just bazel-bin. target_include_directories( tensorflow_cc INTERFACE diff --git a/tensorflow_cc/cmake/ProtobufExternal.cmake b/tensorflow_cc/cmake/ProtobufExternal.cmake new file mode 100644 index 0000000..3957181 --- /dev/null +++ b/tensorflow_cc/cmake/ProtobufExternal.cmake @@ -0,0 +1,31 @@ +# Find the proper protobuf archive url. +file(DOWNLOAD + "https://raw.githubusercontent.com/tensorflow/tensorflow/${TENSORFLOW_TAG}/tensorflow/workspace2.bzl" + "${CMAKE_CURRENT_BINARY_DIR}/tmp/workspace2.bzl" +) +file(READ + "${CMAKE_CURRENT_BINARY_DIR}/tmp/workspace2.bzl" + workspace2_str +) +string(REGEX MATCH + "https://github.com/protocolbuffers/protobuf/archive/v[.0-9]+.zip" + PROTOBUF_ARCHIVE + "${workspace2_str}" +) +message("Will build Protobuf from '${PROTOBUF_ARCHIVE}'.") + +ExternalProject_Add( + protobuf-external + PREFIX protobuf + URL "${PROTOBUF_ARCHIVE}" + CMAKE_CACHE_ARGS + "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" + "-Dprotobuf_BUILD_TESTS:BOOL=OFF" + "-Dprotobuf_BUILD_EXAMPLES:BOOL=OFF" + "-DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}" + SOURCE_SUBDIR cmake + INSTALL_COMMAND "" +) + +install(SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf-external-build/cmake_install.cmake") + diff --git a/tensorflow_cc/cmake/TensorflowBase.cmake b/tensorflow_cc/cmake/TensorflowBase.cmake index 0e29601..74d2478 100644 --- a/tensorflow_cc/cmake/TensorflowBase.cmake +++ b/tensorflow_cc/cmake/TensorflowBase.cmake @@ -11,7 +11,8 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 UPDATE_COMMAND "" CONFIGURE_COMMAND "" - BUILD_COMMAND "${CMAKE_CURRENT_BINARY_DIR}/build_tensorflow.sh" + BUILD_COMMAND echo "*" > "${CMAKE_CURRENT_BINARY_DIR}/tensorflow/.bazelversion" + COMMAND "${CMAKE_CURRENT_BINARY_DIR}/build_tensorflow.sh" INSTALL_COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/cmake/copy_links.sh" bazel-bin # The include folders are sometimes contained under bazel-bin/bin/ and sometimes just bazel-bin. # Later, we include both so let's make sure they both exist so that CMake does not complain. diff --git a/tensorflow_cc/cmake/build_tensorflow.sh.in b/tensorflow_cc/cmake/build_tensorflow.sh.in index a425f9c..3234a50 100755 --- a/tensorflow_cc/cmake/build_tensorflow.sh.in +++ b/tensorflow_cc/cmake/build_tensorflow.sh.in @@ -64,25 +64,17 @@ if [ "$cuda_allowed" == true ] && [ "$cuda_available" == true ]; then export TF_CUDNN_VERSION="$(find /opt /usr -name 'libcudnn.so.*' -path '*/cuda*' | tail -n1 | sed -r 's/^.*\.so\.//')" # choose the right version of CUDA compiler - if [ -z "$GCC_HOST_COMPILER_PATH" ]; then - if hash gcc-11 2>/dev/null && version_gt 11.1 `gcc-11 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-11"} + if [ -z "${GCC_HOST_COMPILER_PATH}" ]; then + if hash gcc-11 2>/dev/null && version_gt 11.4 `gcc-11 -dumpversion`; then + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-11`} elif hash gcc-10 2>/dev/null && version_gt 10.3 `gcc-10 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-10"} + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-10`} elif hash gcc-9 2>/dev/null && version_gt 9.4 `gcc-9 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-9"} + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-9`} elif hash gcc-8 2>/dev/null && version_gt 8.5 `gcc-8 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-8"} - elif hash gcc-7 2>/dev/null && version_gt 7.5 `gcc-7 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-7"} - elif hash gcc-6 2>/dev/null && version_gt 6.4 `gcc-6 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-6"} - elif hash gcc-5 2>/dev/null && version_gt 5.5 `gcc-5 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-5"} - elif hash gcc-4 2>/dev/null && version_gt 4.9 `gcc-4 -dumpversion`; then - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc-4"} + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc-8`} else - export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-"/usr/bin/gcc"} + export GCC_HOST_COMPILER_PATH=${GCC_HOST_COMPILER_PATH:-`which gcc`} fi fi