From 2a369c95b64fb84d9a0d94a1e83b1bc2e3c07ecc Mon Sep 17 00:00:00 2001 From: Kirthi Shankar Sivamani Date: Tue, 30 Jul 2024 22:52:24 +0000 Subject: [PATCH 1/6] Use pyproject.toml to specify build requirements Signed-off-by: Kirthi Shankar Sivamani --- build_tools/utils.py | 2 +- pyproject.toml | 5 +++++ setup.py | 16 +++------------- 3 files changed, 9 insertions(+), 14 deletions(-) create mode 100644 pyproject.toml diff --git a/build_tools/utils.py b/build_tools/utils.py index 3230ad35bf..da9ffab01e 100644 --- a/build_tools/utils.py +++ b/build_tools/utils.py @@ -270,7 +270,7 @@ def install_and_import(package): def uninstall_te_fw_packages(): - subprocess.check_call( + subprocess.run( [ sys.executable, "-m", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..c2bd4a2136 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[build-system] +requires = ["setuptools >= 61.0", "cmake>=3.21", "pybind11", "ninja"] + +# Use legacy backend to import local packages in setup.py +build-backend = "setuptools.build_meta:__legacy__" diff --git a/setup.py b/setup.py index 6a8bae2793..b9e78399f0 100644 --- a/setup.py +++ b/setup.py @@ -54,14 +54,13 @@ def setup_common_extension() -> CMakeExtension: ) -def setup_requirements() -> Tuple[List[str], List[str], List[str]]: +def setup_requirements() -> Tuple[List[str], List[str]]: """Setup Python dependencies Returns dependencies for build, runtime, and testing. """ # Common requirements - setup_reqs: List[str] = [] install_reqs: List[str] = [ "pydantic", "importlib-metadata>=1.0", @@ -69,20 +68,12 @@ def setup_requirements() -> Tuple[List[str], List[str], List[str]]: ] test_reqs: List[str] = ["pytest>=8.2.1"] - # Requirements that may be installed outside of Python - if not found_cmake(): - setup_reqs.append("cmake>=3.21") - if not found_ninja(): - setup_reqs.append("ninja") - if not found_pybind11(): - setup_reqs.append("pybind11") - - return [remove_dups(reqs) for reqs in [setup_reqs, install_reqs, test_reqs]] + return [remove_dups(reqs) for reqs in [install_reqs, test_reqs]] if __name__ == "__main__": # Dependencies - setup_requires, install_requires, test_requires = setup_requirements() + install_requires, test_requires = setup_requirements() __version__ = te_version() @@ -150,7 +141,6 @@ def setup_requirements() -> Tuple[List[str], List[str], List[str]]: "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - setup_requires=setup_requires, install_requires=install_requires, license_files=("LICENSE",), include_package_data=True, From ee91ba5dc4162d96f8898b5b23567286ad9e5231 Mon Sep 17 00:00:00 2001 From: Kirthi Shankar Sivamani Date: Wed, 31 Jul 2024 05:21:32 +0000 Subject: [PATCH 2/6] fix Signed-off-by: Kirthi Shankar Sivamani --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index b9e78399f0..7fd88aa26e 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ found_pybind11, remove_dups, get_frameworks, - install_and_import, uninstall_te_fw_packages, ) from build_tools.te_version import te_version @@ -36,7 +35,6 @@ elif "paddle" in frameworks: from paddle.utils.cpp_extension import BuildExtension elif "jax" in frameworks: - install_and_import("pybind11[global]") from pybind11.setup_helpers import build_ext as BuildExtension From 69db5bd37dbe0a9f799cf3ef16ffd2a5237b0fd7 Mon Sep 17 00:00:00 2001 From: Kirthi Shankar Sivamani Date: Wed, 31 Jul 2024 21:36:57 +0000 Subject: [PATCH 3/6] New approach to avoid build iso Signed-off-by: Kirthi Shankar Sivamani --- build_tools/utils.py | 11 +++++------ setup.py | 7 +++++++ transformer_engine/jax/setup.py | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/build_tools/utils.py b/build_tools/utils.py index da9ffab01e..4af7d8e076 100644 --- a/build_tools/utils.py +++ b/build_tools/utils.py @@ -11,7 +11,6 @@ import shutil import subprocess import sys -import importlib from pathlib import Path from subprocess import CalledProcessError from typing import List, Optional, Tuple @@ -262,11 +261,11 @@ def copy_common_headers(te_src, dst): shutil.copy(file_path, new_path) -def install_and_import(package): - """Install a package via pip (if not already installed) and import into globals.""" - main_package = package.split("[")[0] - subprocess.check_call([sys.executable, "-m", "pip", "install", package]) - globals()[main_package] = importlib.import_module(main_package) +def install_packages(packages): + """Install a package via pip (if not already installed).""" + for package in packages: + main_package = package.split("[")[0] + subprocess.run([sys.executable, "-m", "pip", "install", package]) def uninstall_te_fw_packages(): diff --git a/setup.py b/setup.py index 7fd88aa26e..5d89819e12 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ remove_dups, get_frameworks, uninstall_te_fw_packages, + install_packages, ) from build_tools.te_version import te_version @@ -28,13 +29,19 @@ from setuptools.command.build_ext import build_ext as BuildExtension +# Same list as build_system.requires in pyproject.toml +install_packages(["setuptools >= 61.0", "cmake>=3.21", "pybind11", "ninja"]) + os.environ["NVTE_PROJECT_BUILDING"] = "1" if "pytorch" in frameworks: + install_packages(["torch>=1.13"]) from torch.utils.cpp_extension import BuildExtension elif "paddle" in frameworks: + install_packages(["paddlepaddle-gpu>=2.6.1"]) from paddle.utils.cpp_extension import BuildExtension elif "jax" in frameworks: + install_packages(["jax"]) from pybind11.setup_helpers import build_ext as BuildExtension diff --git a/transformer_engine/jax/setup.py b/transformer_engine/jax/setup.py index c2219e3ba9..b8dfefc055 100644 --- a/transformer_engine/jax/setup.py +++ b/transformer_engine/jax/setup.py @@ -29,11 +29,11 @@ from build_tools.build_ext import get_build_ext -from build_tools.utils import copy_common_headers, install_and_import +from build_tools.utils import copy_common_headers, install_packages from build_tools.te_version import te_version from build_tools.jax import setup_jax_extension -install_and_import("pybind11") +install_packages("pybind11") from pybind11.setup_helpers import build_ext as BuildExtension os.environ["NVTE_PROJECT_BUILDING"] = "1" From 474053ad305a923cc5341359e9ba369110d8e964 Mon Sep 17 00:00:00 2001 From: Kirthi Shankar Sivamani Date: Wed, 31 Jul 2024 21:48:58 +0000 Subject: [PATCH 4/6] fix Signed-off-by: Kirthi Shankar Sivamani --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fb7ab345d1..6ca71d7173 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: with: submodules: recursive - name: 'Build' - run: pip install . -v --no-deps + run: pip install . -v env: NVTE_FRAMEWORK: pytorch MAX_JOBS: 1 From ad9e33bc1fc57693e77f39b78de07ecebe307a94 Mon Sep 17 00:00:00 2001 From: Kirthi Shankar Sivamani Date: Thu, 1 Aug 2024 07:34:45 +0000 Subject: [PATCH 5/6] no build isolation Signed-off-by: Kirthi Shankar Sivamani --- .github/workflows/build.yml | 6 +++--- README.rst | 2 +- build_tools/utils.py | 6 ++++-- docs/installation.rst | 14 +++++++------- pyproject.toml | 2 +- setup.py | 6 ++---- transformer_engine/jax/setup.py | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ca71d7173..f6c1bc316a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: with: submodules: recursive - name: 'Build' - run: pip install . -v + run: pip install . -v --no-build-isolation env: NVTE_FRAMEWORK: pytorch MAX_JOBS: 1 @@ -61,7 +61,7 @@ jobs: with: submodules: recursive - name: 'Build' - run: pip install . -v + run: pip install . -v --no-build-isolation env: NVTE_FRAMEWORK: jax - name: 'Sanity check' @@ -81,7 +81,7 @@ jobs: run: | apt-get update apt-get install -y libgoogle-glog-dev - pip install . -v + pip install . -v --no-build-isolation env: NVTE_FRAMEWORK: paddle - name: 'Sanity check' diff --git a/README.rst b/README.rst index 085c91ca49..febeefb425 100644 --- a/README.rst +++ b/README.rst @@ -172,7 +172,7 @@ To install the latest stable version of Transformer Engine, .. code-block:: bash - pip install git+https://github.com/NVIDIA/TransformerEngine.git@stable + pip install --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@stable This will automatically detect if any supported deep learning frameworks are installed and build Transformer Engine support for them. To explicitly specify frameworks, set the environment variable NVTE_FRAMEWORK to a comma-separated list (e.g. NVTE_FRAMEWORK=jax,pytorch). diff --git a/build_tools/utils.py b/build_tools/utils.py index 4af7d8e076..f81ff379a6 100644 --- a/build_tools/utils.py +++ b/build_tools/utils.py @@ -11,6 +11,7 @@ import shutil import subprocess import sys +import importlib from pathlib import Path from subprocess import CalledProcessError from typing import List, Optional, Tuple @@ -264,8 +265,9 @@ def copy_common_headers(te_src, dst): def install_packages(packages): """Install a package via pip (if not already installed).""" for package in packages: - main_package = package.split("[")[0] - subprocess.run([sys.executable, "-m", "pip", "install", package]) + main_package = package.split(">")[0] + subprocess.run([sys.executable, "-m", "pip", "install", package], check=True) + globals()[main_package] = importlib.import_module(main_package) def uninstall_te_fw_packages(): diff --git a/docs/installation.rst b/docs/installation.rst index 5dd10a79d1..c9846fb658 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -44,7 +44,7 @@ Execute the following command to install the latest stable version of Transforme .. code-block:: bash - pip install git+https://github.com/NVIDIA/TransformerEngine.git@stable + pip install --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@stable This will automatically detect if any supported deep learning frameworks are installed and build Transformer Engine support for them. To explicitly specify frameworks, set the environment variable `NVTE_FRAMEWORK` to a comma-separated list (e.g. `NVTE_FRAMEWORK=jax,pytorch`). @@ -61,7 +61,7 @@ Execute the following command to install the latest development build of Transfo .. code-block:: bash - pip install git+https://github.com/NVIDIA/TransformerEngine.git@main + pip install --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@main This will automatically detect if any supported deep learning frameworks are installed and build Transformer Engine support for them. To explicitly specify frameworks, set the environment variable `NVTE_FRAMEWORK` to a comma-separated list (e.g. `NVTE_FRAMEWORK=jax,pytorch`). To only build the framework-agnostic C++ API, set `NVTE_FRAMEWORK=none`. @@ -69,7 +69,7 @@ In order to install a specific PR, execute after changing NNN to the PR number: .. code-block:: bash - pip install git+https://github.com/NVIDIA/TransformerEngine.git@refs/pull/NNN/merge + pip install --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@refs/pull/NNN/merge Installation (from source) @@ -83,8 +83,8 @@ Execute the following commands to install Transformer Engine from source: git clone --branch stable --recursive https://github.com/NVIDIA/TransformerEngine.git cd TransformerEngine - export NVTE_FRAMEWORK=pytorch # Optionally set framework - pip install . # Build and install + export NVTE_FRAMEWORK=pytorch # Optionally set framework + pip install . --no-build-isolation # Build and install If the Git repository has already been cloned, make sure to also clone the submodules: @@ -96,10 +96,10 @@ Extra dependencies for testing can be installed by setting the "test" option: .. code-block:: bash - pip install .[test] + pip install .[test] --no-build-isolation To build the C++ extensions with debug symbols, e.g. with the `-g` flag: .. code-block:: bash - pip install . --global-option=--debug + pip install . --global-option=--debug --no-build-isolation diff --git a/pyproject.toml b/pyproject.toml index c2bd4a2136..3a176c2a5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools >= 61.0", "cmake>=3.21", "pybind11", "ninja"] +requires = ["setuptools>=61.0", "cmake>=3.21", "pybind11", "ninja", "pip"] # Use legacy backend to import local packages in setup.py build-backend = "setuptools.build_meta:__legacy__" diff --git a/setup.py b/setup.py index 5d89819e12..58152cbb50 100644 --- a/setup.py +++ b/setup.py @@ -30,18 +30,15 @@ from setuptools.command.build_ext import build_ext as BuildExtension # Same list as build_system.requires in pyproject.toml -install_packages(["setuptools >= 61.0", "cmake>=3.21", "pybind11", "ninja"]) +install_packages(["setuptools>=61.0", "cmake>=3.21", "pybind11", "ninja", "pip"]) os.environ["NVTE_PROJECT_BUILDING"] = "1" if "pytorch" in frameworks: - install_packages(["torch>=1.13"]) from torch.utils.cpp_extension import BuildExtension elif "paddle" in frameworks: - install_packages(["paddlepaddle-gpu>=2.6.1"]) from paddle.utils.cpp_extension import BuildExtension elif "jax" in frameworks: - install_packages(["jax"]) from pybind11.setup_helpers import build_ext as BuildExtension @@ -78,6 +75,7 @@ def setup_requirements() -> Tuple[List[str], List[str]]: if __name__ == "__main__": # Dependencies + install_requires, test_requires = setup_requirements() __version__ = te_version() diff --git a/transformer_engine/jax/setup.py b/transformer_engine/jax/setup.py index b8dfefc055..9b29654f7b 100644 --- a/transformer_engine/jax/setup.py +++ b/transformer_engine/jax/setup.py @@ -33,7 +33,7 @@ from build_tools.te_version import te_version from build_tools.jax import setup_jax_extension -install_packages("pybind11") +install_packages(["pybind11"]) from pybind11.setup_helpers import build_ext as BuildExtension os.environ["NVTE_PROJECT_BUILDING"] = "1" From d450de5dcf526e547f446e0f964b8c12f510a78c Mon Sep 17 00:00:00 2001 From: Kirthi Shankar Sivamani Date: Thu, 1 Aug 2024 21:22:58 +0000 Subject: [PATCH 6/6] Fixes Signed-off-by: Kirthi Shankar Sivamani --- requirements.txt | 9 +++++++++ setup.py | 4 +--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..21e2ee6390 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +# Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See LICENSE for license information. + +setuptools>=61.0 +cmake>=3.21 +pybind11 +ninja +pip diff --git a/setup.py b/setup.py index 58152cbb50..dd9d4d40ab 100644 --- a/setup.py +++ b/setup.py @@ -29,9 +29,7 @@ from setuptools.command.build_ext import build_ext as BuildExtension -# Same list as build_system.requires in pyproject.toml -install_packages(["setuptools>=61.0", "cmake>=3.21", "pybind11", "ninja", "pip"]) - +install_packages(["pybind11"]) os.environ["NVTE_PROJECT_BUILDING"] = "1" if "pytorch" in frameworks: