From 8aa71bd9f28ed682443f1d3f35721f1ae3b52863 Mon Sep 17 00:00:00 2001 From: albanD Date: Wed, 15 Nov 2023 09:40:48 -0500 Subject: [PATCH] Changes needed for core enablement of 3.12 binary wheels (#1586) --- aarch64_linux/aarch64_ci_setup.sh | 7 +++++-- conda/pytorch-nightly/meta.yaml | 9 ++++++--- manywheel/build_common.sh | 3 +++ manywheel/build_cuda.sh | 4 +++- manywheel/build_rocm.sh | 4 +++- wheel/build_wheel.sh | 13 ++++++++++++- windows/condaenv.bat | 1 + windows/internal/smoke_test.bat | 1 + 8 files changed, 34 insertions(+), 8 deletions(-) diff --git a/aarch64_linux/aarch64_ci_setup.sh b/aarch64_linux/aarch64_ci_setup.sh index 6d2d780fe..ace6a85fb 100755 --- a/aarch64_linux/aarch64_ci_setup.sh +++ b/aarch64_linux/aarch64_ci_setup.sh @@ -19,7 +19,10 @@ curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/late chmod +x /mambaforge.sh /mambaforge.sh -b -p /opt/conda rm /mambaforge.sh -/opt/conda/bin/conda config --set ssl_verify False -/opt/conda/bin/conda install -y -c conda-forge python=${DESIRED_PYTHON} numpy pyyaml setuptools patchelf pygit2 openblas ninja scons +source /opt/conda/etc/profile.d/conda.sh +conda config --set ssl_verify False +conda create -y -c conda-forge -n aarch64_env python=${DESIRED_PYTHON} +conda activate aarch64_env +conda install -y -c conda-forge numpy==1.26.0 pyyaml==6.0.1 patchelf==0.17.2 pygit2==1.13.2 openblas==0.3.24 ninja==1.11.1 scons==4.5.2 python --version conda --version diff --git a/conda/pytorch-nightly/meta.yaml b/conda/pytorch-nightly/meta.yaml index 4a03c3d05..882f13da0 100644 --- a/conda/pytorch-nightly/meta.yaml +++ b/conda/pytorch-nightly/meta.yaml @@ -21,7 +21,8 @@ requirements: - pyyaml {% if cross_compile_arm64 == 0 %} - mkl-include # [x86_64] - - mkl=2020.2 # [x86_64 and not win] + - mkl=2020.2 # [py <= 311 and x86_64 and not win] + - mkl=2023.1 # [py >= 312 and x86_64 and not win] - mkl=2021.4 # [x86_64 and win] {% endif %} - typing_extensions @@ -29,9 +30,11 @@ requirements: - libuv # [win] - numpy=1.19 # [py <= 39] - numpy=1.21.5 # [py == 310] - - numpy=1.23.5 # [py >= 311] + - numpy=1.23.5 # [py == 311] + - numpy=1.26.1 # [py >= 312] - openssl=1.1.1l # [py >= 38 and py <= 310 and linux] - - openssl=1.1.1s # [py >= 311 and linux] + - openssl=1.1.1s # [py == 311 and linux] + - openssl=3.1.4 # [py >= 312 and linux] {{ environ.get('PYTORCH_LLVM_PACKAGE', ' - llvmdev=9') }} {{ environ.get('MAGMA_PACKAGE', '') }} diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh index f4794b719..31f188d94 100644 --- a/manywheel/build_common.sh +++ b/manywheel/build_common.sh @@ -131,6 +131,9 @@ case ${DESIRED_PYTHON} in cp311*) retry pip install -q numpy==1.23.1 ;; + cp312*) + retry pip install -q numpy==1.26.1 + ;; # Should catch 3.9+ *) retry pip install -q numpy==1.19.4 diff --git a/manywheel/build_cuda.sh b/manywheel/build_cuda.sh index c59cbca16..31f4e263b 100644 --- a/manywheel/build_cuda.sh +++ b/manywheel/build_cuda.sh @@ -263,7 +263,9 @@ else fi # TODO: Remove me when Triton has a proper release channel -if [[ $(uname) == "Linux" ]]; then +# No triton dependency for now on 3.12 since we don't have binaries for it +# and torch.compile doesn't work. +if [[ $(uname) == "Linux" && "$DESIRED_PYTHON" != "3.12" ]]; then TRITON_SHORTHASH=$(cut -c1-10 $PYTORCH_ROOT/.github/ci_commit_pins/triton.txt) if [[ -z "$PYTORCH_EXTRA_INSTALL_REQUIREMENTS" ]]; then diff --git a/manywheel/build_rocm.sh b/manywheel/build_rocm.sh index 0fed5970b..4fbca7697 100755 --- a/manywheel/build_rocm.sh +++ b/manywheel/build_rocm.sh @@ -222,7 +222,9 @@ if [[ $ROCM_INT -ge 50600 ]]; then fi # Add triton install dependency -if [[ $(uname) == "Linux" ]]; then +# No triton dependency for now on 3.12 since we don't have binaries for it +# and torch.compile doesn't work. +if [[ $(uname) == "Linux" && "$DESIRED_PYTHON" != "3.12" ]]; then TRITON_SHORTHASH=$(cut -c1-10 $PYTORCH_ROOT/.ci/docker/ci_commit_pins/triton-rocm.txt) TRITON_VERSION=$(cat $PYTORCH_ROOT/.ci/docker/triton_version.txt) diff --git a/wheel/build_wheel.sh b/wheel/build_wheel.sh index c3c45cf42..1186bc56a 100755 --- a/wheel/build_wheel.sh +++ b/wheel/build_wheel.sh @@ -136,23 +136,33 @@ export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} SETUPTOOLS_PINNED_VERSION="=46.0.0" PYYAML_PINNED_VERSION="=5.3" EXTRA_CONDA_INSTALL_FLAGS="" -case ${desired_python} in +case $desired_python in + 3.12) + echo "Using 3.12 deps" + SETUPTOOLS_PINNED_VERSION=">=68.0.0" + PYYAML_PINNED_VERSION=">=6.0.1" + NUMPY_PINNED_VERSION="==1.26.0" + ;; 3.11) + echo "Using 3.11 deps" SETUPTOOLS_PINNED_VERSION=">=46.0.0" PYYAML_PINNED_VERSION=">=5.3" NUMPY_PINNED_VERSION="==1.23.5" ;; 3.10) + echo "Using 3.10 deps" SETUPTOOLS_PINNED_VERSION=">=46.0.0" PYYAML_PINNED_VERSION=">=5.3" NUMPY_PINNED_VERSION="=1.21.2" ;; 3.9) + echo "Using 3.9 deps" SETUPTOOLS_PINNED_VERSION=">=46.0.0" PYYAML_PINNED_VERSION=">=5.3" NUMPY_PINNED_VERSION="=1.19" ;; 3.8) + echo "Using 3.8 deps" if [[ "$(uname -m)" == "arm64" ]]; then SETUPTOOLS_PINNED_VERSION=">=46.0.0" PYYAML_PINNED_VERSION=">=5.3" @@ -162,6 +172,7 @@ case ${desired_python} in fi ;; *) + echo "Using default deps" NUMPY_PINNED_VERSION="=1.11.3" ;; esac diff --git a/windows/condaenv.bat b/windows/condaenv.bat index 464eeb01c..cf1b2c865 100644 --- a/windows/condaenv.bat +++ b/windows/condaenv.bat @@ -14,6 +14,7 @@ FOR %%v IN (%DESIRED_PYTHON%) DO ( if "%%v" == "3.9" call conda create -n py!PYTHON_VERSION_STR! -y -q numpy>=1.11 "mkl=2020.2" pyyaml boto3 cmake ninja typing_extensions python=%%v if "%%v" == "3.10" call conda create -n py!PYTHON_VERSION_STR! -y -q -c=conda-forge numpy=1.21.3 "mkl=2020.2" pyyaml boto3 "cmake=3.19.6" ninja typing_extensions python=%%v if "%%v" == "3.11" call conda create -n py!PYTHON_VERSION_STR! -y -q -c=conda-forge numpy=1.23.4 "mkl=2020.2" pyyaml boto3 "cmake=3.19.6" ninja typing_extensions python=%%v + if "%%v" == "3.12" call conda create -n py!PYTHON_VERSION_STR! -y -q -c=conda-forge numpy=1.26.0 "mkl=2023.1" pyyaml boto3 "cmake=3.19.6" ninja typing_extensions python=%%v if "%%v" == "3" call conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl=2020.2" pyyaml boto3 cmake ninja typing_extensions python=%%v ) endlocal diff --git a/windows/internal/smoke_test.bat b/windows/internal/smoke_test.bat index ad276b992..1ade2cbda 100644 --- a/windows/internal/smoke_test.bat +++ b/windows/internal/smoke_test.bat @@ -30,6 +30,7 @@ exit /b 1 echo "install wheel package" set PYTHON_INSTALLER_URL= +if "%DESIRED_PYTHON%" == "3.12" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.12.0/python-3.12.0-amd64.exe" if "%DESIRED_PYTHON%" == "3.11" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.11.0/python-3.11.0-amd64.exe" if "%DESIRED_PYTHON%" == "3.10" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe" if "%DESIRED_PYTHON%" == "3.9" set "PYTHON_INSTALLER_URL=https://www.python.org/ftp/python/3.9.0/python-3.9.0-amd64.exe"