From 2cbe852b58ef7b366d552e4b101399619710f1b6 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 29 Jan 2024 17:52:23 -0500 Subject: [PATCH 1/2] allow disabling TensorFlow backend during Python installation Fix #3120. Signed-off-by: Jinzhe Zeng --- backend/find_tensorflow.py | 6 ++++++ backend/read_env.py | 24 +++++++++++++++++------- deepmd/tf/env.py | 6 ++++++ doc/install/install-from-source.md | 15 +++++++++++++-- source/CMakeLists.txt | 4 +++- source/config/CMakeLists.txt | 14 ++++++++++++++ source/config/run_config.ini | 2 ++ source/lib/src/gpu/CMakeLists.txt | 6 ++++-- 8 files changed, 65 insertions(+), 12 deletions(-) diff --git a/backend/find_tensorflow.py b/backend/find_tensorflow.py index 32ae62469c..083e2673f7 100644 --- a/backend/find_tensorflow.py +++ b/backend/find_tensorflow.py @@ -127,6 +127,12 @@ def get_tf_requirement(tf_version: str = "") -> dict: dict TensorFlow requirement, including cpu and gpu. """ + if tf_version is None: + return { + "cpu": [], + "gpu": [], + "mpi": [], + } if tf_version == "": tf_version = os.environ.get("TENSORFLOW_VERSION", "") diff --git a/backend/read_env.py b/backend/read_env.py index 2cf433181a..bee5d607e3 100644 --- a/backend/read_env.py +++ b/backend/read_env.py @@ -80,16 +80,26 @@ def get_argument_from_env() -> Tuple[str, list, list, dict, str]: cmake_args.append("-DENABLE_IPI:BOOL=TRUE") extra_scripts["dp_ipi"] = "deepmd.tf.entrypoints.ipi:dp_ipi" - tf_install_dir, _ = find_tensorflow() - tf_version = get_tf_version(tf_install_dir) - if tf_version == "" or Version(tf_version) >= Version("2.12"): - find_libpython_requires = [] + if os.environ.get("DP_ENABLE_TENSORFLOW", "1") == "1": + tf_install_dir, _ = find_tensorflow() + tf_version = get_tf_version(tf_install_dir) + if tf_version == "" or Version(tf_version) >= Version("2.12"): + find_libpython_requires = [] + else: + find_libpython_requires = ["find_libpython"] + cmake_args.extend( + [ + "-DENABLE_TENSORFLOW=ON", + f"-DTENSORFLOW_VERSION={tf_version}", + f"-DTENSORFLOW_ROOT:PATH={tf_install_dir}", + ] + ) else: - find_libpython_requires = ["find_libpython"] - cmake_args.append(f"-DTENSORFLOW_VERSION={tf_version}") + find_libpython_requires = [] + cmake_args.append("-DENABLE_TENSORFLOW=OFF") + tf_version = None cmake_args = [ - f"-DTENSORFLOW_ROOT:PATH={tf_install_dir}", "-DBUILD_PY_IF:BOOL=TRUE", *cmake_args, ] diff --git a/deepmd/tf/env.py b/deepmd/tf/env.py index da03631689..14569d1dd5 100644 --- a/deepmd/tf/env.py +++ b/deepmd/tf/env.py @@ -472,6 +472,12 @@ def _get_package_constants( GLOBAL_CONFIG = _get_package_constants() +print(GLOBAL_CONFIG) +if GLOBAL_CONFIG["enable_tensorflow"] == "0": + raise RuntimeError( + "TensorFlow backend is not built. To enable it, " + "set the environmental variable DP_ENABLE_TENSORFLOW=1." + ) MODEL_VERSION = GLOBAL_CONFIG["model_version"] TF_VERSION = GLOBAL_CONFIG["tf_version"] TF_CXX11_ABI_FLAG = int(GLOBAL_CONFIG["tf_cxx11_abi_flag"]) diff --git a/doc/install/install-from-source.md b/doc/install/install-from-source.md index ae1509f2ca..389cc78c9f 100644 --- a/doc/install/install-from-source.md +++ b/doc/install/install-from-source.md @@ -90,7 +90,17 @@ Check the compiler version on your machine gcc --version ``` -The compiler GCC 4.8 or later is supported in the DeePMD-kit. Note that TensorFlow may have specific requirements for the compiler version to support the C++ standard version and [`_GLIBCXX_USE_CXX11_ABI`](https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html) used by TensorFlow. It is recommended to use [the same compiler version as TensorFlow](https://www.tensorflow.org/install/source#tested_build_configurations), which can be printed by `python -c "import tensorflow;print(tensorflow.version.COMPILER_VERSION)"`. +The compiler GCC 4.8 or later is supported in the DeePMD-kit. + +::::{tab-set} + +:::{tab-item} TensorFlow {{ tensorflow_icon }} + +Note that TensorFlow may have specific requirements for the compiler version to support the C++ standard version and [`_GLIBCXX_USE_CXX11_ABI`](https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html) used by TensorFlow. It is recommended to use [the same compiler version as TensorFlow](https://www.tensorflow.org/install/source#tested_build_configurations), which can be printed by `python -c "import tensorflow;print(tensorflow.version.COMPILER_VERSION)"`. + +::: + +:::: Execute ```bash @@ -105,7 +115,8 @@ One may set the following environment variables before executing `pip`: | DP_VARIANT | `cpu`, `cuda`, `rocm` | `cpu` | Build CPU variant or GPU variant with CUDA or ROCM support. | | CUDAToolkit_ROOT | Path | Detected automatically | The path to the CUDA toolkit directory. CUDA 9.0 or later is supported. NVCC is required. | | ROCM_ROOT | Path | Detected automatically | The path to the ROCM toolkit directory. | -| TENSORFLOW_ROOT | Path | Detected automatically | The path to TensorFlow Python library. By default the installer only finds TensorFlow under user site-package directory (`site.getusersitepackages()`) or system site-package directory (`sysconfig.get_path("purelib")`) due to limitation of [PEP-517](https://peps.python.org/pep-0517/). If not found, the latest TensorFlow (or the environment variable `TENSORFLOW_VERSION` if given) from PyPI will be built against.| +| DP_ENABLE_TENSORFLOW | 0, 1 | 1 | {{ tensorflow_icon }} Enable the TensorFlow backend. +| TENSORFLOW_ROOT | Path | Detected automatically | {{ tensorflow_icon }} The path to TensorFlow Python library. By default the installer only finds TensorFlow under user site-package directory (`site.getusersitepackages()`) or system site-package directory (`sysconfig.get_path("purelib")`) due to limitation of [PEP-517](https://peps.python.org/pep-0517/). If not found, the latest TensorFlow (or the environment variable `TENSORFLOW_VERSION` if given) from PyPI will be built against.| | DP_ENABLE_NATIVE_OPTIMIZATION | 0, 1 | 0 | Enable compilation optimization for the native machine's CPU type. Do not enable it if generated code will run on different CPUs. | | CMAKE_ARGS | str | - | Additional CMake arguments | | <LANG>FLAGS (``=`CXX`, `CUDA` or `HIP`) | str | - | Default compilation flags to be used when compiling `` files. See [CMake documentation](https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_FLAGS.html). | diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index c273bc9263..747bdf30f7 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -165,7 +165,9 @@ if(NOT DEEPMD_C_ROOT) if(ENABLE_PYTORCH) message(STATUS "- PyTorch") endif() - if(NOT ENABLE_TENSORFLOW AND NOT ENABLE_PYTORCH) + if(NOT ENABLE_TENSORFLOW + AND NOT ENABLE_PYTORCH + AND NOT BUILD_PY_IF) message(FATAL_ERROR "No backend is enabled.") endif() endif() diff --git a/source/config/CMakeLists.txt b/source/config/CMakeLists.txt index 5473b91f29..b1ce17566f 100644 --- a/source/config/CMakeLists.txt +++ b/source/config/CMakeLists.txt @@ -1,5 +1,19 @@ # config +# cmake will treat true, false, on, off, 1, 0 as booleans we hope an easy way to +# check it +if(ENABLE_TENSORFLOW) + set(ENABLE_TENSORFLOW 1) +else() + set(ENABLE_TENSORFLOW 0) +endif() + +if(ENABLE_PYTORCH) + set(ENABLE_PYTORCH 1) +else() + set(ENABLE_PYTORCH 0) +endif() + configure_file("run_config.ini" "${CMAKE_CURRENT_BINARY_DIR}/run_config.ini" @ONLY) diff --git a/source/config/run_config.ini b/source/config/run_config.ini index 3f0a7a33a8..11f4100e61 100644 --- a/source/config/run_config.ini +++ b/source/config/run_config.ini @@ -4,6 +4,8 @@ GIT_SUMM = @GIT_SUMM@ GIT_HASH = @GIT_HASH@ GIT_DATE = @GIT_DATE@ GIT_BRANCH = @GIT_BRANCH@ +ENABLE_TENSORFLOW = @ENABLE_TENSORFLOW@ +ENABLE_PYTORCH = @ENABLE_PYTORCH@ TF_INCLUDE_DIR = @TensorFlow_INCLUDE_DIRS@ TF_LIBS = @TensorFlow_LIBRARY@ TF_VERSION = @TENSORFLOW_VERSION@ diff --git a/source/lib/src/gpu/CMakeLists.txt b/source/lib/src/gpu/CMakeLists.txt index 3bd24cc620..804e1c0506 100644 --- a/source/lib/src/gpu/CMakeLists.txt +++ b/source/lib/src/gpu/CMakeLists.txt @@ -10,8 +10,10 @@ if(USE_CUDA_TOOLKIT) endif() enable_language(CUDA) set(CMAKE_CUDA_STANDARD 11) - add_compile_definitions( - "$<$:_GLIBCXX_USE_CXX11_ABI=${OP_CXX_ABI}>") + if(DEFINED OP_CXX_ABI) + add_compile_definitions( + "$<$:_GLIBCXX_USE_CXX11_ABI=${OP_CXX_ABI}>") + endif() find_package(CUDAToolkit REQUIRED) From 814a5492eade44f7fc098989ce189f48a60c44b7 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 29 Jan 2024 17:53:51 -0500 Subject: [PATCH 2/2] remove debug line Signed-off-by: Jinzhe Zeng --- deepmd/tf/env.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deepmd/tf/env.py b/deepmd/tf/env.py index 14569d1dd5..eada2774d3 100644 --- a/deepmd/tf/env.py +++ b/deepmd/tf/env.py @@ -472,7 +472,6 @@ def _get_package_constants( GLOBAL_CONFIG = _get_package_constants() -print(GLOBAL_CONFIG) if GLOBAL_CONFIG["enable_tensorflow"] == "0": raise RuntimeError( "TensorFlow backend is not built. To enable it, "