Release using versions (#1615)

* Changes needed for core enablement of 3.12 binary wheels (#1586) * Fix aarch64 build on 3.8 (#1593) * Add some more validation checks for torch.linalg.eigh and torch.compile (#1580) * Add some more validation checks for torch.linalg.eigh and torch.compile * Update test * Also update smoke_test.py * Fix lint * Revert "Add some more validation checks for torch.linalg.eigh and torch.compile (#1580)" (#1594) This reverts commit 4c7fa06. * Release validations using release version matrix (#1611) * Release pypi prep change (#1587) * [aarch64] Release pypi prep script change for aarch64 builds * Release versions for testing Testing calling version (#1588) Upstream/release validations (#1589) * Testing calling version * add release matrix Upstream/release validations (#1590) * Testing calling version * add release matrix * test test (#1591) test (#1592) Release v1 (#1595) * test * test Release v1 (#1596) * test * test * test test (#1597) Test versions validations (#1598) * test * basedir Test versions validations (#1599) * test * basedir * test test (#1600) * test * test Add release versions everywhere (#1601) * test * test * test * test test (#1602) Test version validations (#1603) * test * test Test version validations (#1604) * test * test * test tests (#1605) More tests nov16 (#1606) * tests * test More tests nov16 (#1607) * tests * test * test More tests nov16 (#1608) * tests * test * test * test More tests nov16 (#1609) * tests * test * test * test * test * fix_lint * fix: typo (#1581) * desired_cuda -> DESIRED_CUDA (#1612) * desired_cuda -> DESIRED_CUDA Found with shellcheck * Update manywheel/build_cuda.sh Co-authored-by: Nikita Shulga <[email protected]> --------- Co-authored-by: Nikita Shulga <[email protected]> * [BE] Cleanup build unused code (#1613) 1. Upload Scripts are not used anymore. We use Github Action upload workflows 2. M1 Builds are now automated 3. build_all.bat run git grep in pytorch and builder - No result * Changes to pypi release promotion scripts introduced for 2.1.0 and 2.1.1 (#1614) * Changes topypi release promotion scripts introduced during 2.1.1 * typo * Pin miniconda version for Windows To Miniconda3-py311_23.9.0-0-Windows-x86_64.exe * test --------- Co-authored-by: albanD <[email protected]> Co-authored-by: Huy Do <[email protected]> Co-authored-by: Luo Bo <[email protected]> Co-authored-by: Sergii Dymchenko <[email protected]> Co-authored-by: Nikita Shulga <[email protected]>
pytorch · Nov 17, 2023 · 0798088 · 0798088
1 parent b83f9f9
commit 0798088
Show file tree

Hide file tree

Showing 19 changed files with 69 additions and 242 deletions.
diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
@@ -12,6 +12,12 @@ else
         TEST_SUFFIX=" --package torchonly"
     fi
 
+    # if RELESE version is passed as parameter - install speific version
+    if [[ ! -z ${RELEASE_VERSION} ]]; then
+          INSTALLATION=${INSTALLATION/"torch "/"torch==${RELEASE_VERSION} "}
+          INSTALLATION=${INSTALLATION/"pytorch "/"pytorch==${RELEASE_VERSION} "}
+    fi
+
     export OLD_PATH=${PATH}
     # Workaround macos-arm64 runners. Issue: https://github.com/pytorch/test-infra/issues/4342
     if [[ ${TARGET_OS} == 'macos-arm64' ]]; then

diff --git a/aarch64_linux/aarch64_ci_setup.sh b/aarch64_linux/aarch64_ci_setup.sh
@@ -19,7 +19,17 @@ curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/late
 chmod +x /mambaforge.sh
 /mambaforge.sh -b -p /opt/conda
 rm /mambaforge.sh
-/opt/conda/bin/conda config --set ssl_verify False
-/opt/conda/bin/conda install -y -c conda-forge python=${DESIRED_PYTHON} numpy pyyaml setuptools patchelf pygit2 openblas ninja scons
+source /opt/conda/etc/profile.d/conda.sh
+conda config --set ssl_verify False
+conda create -y -c conda-forge -n aarch64_env python=${DESIRED_PYTHON}
+conda activate aarch64_env
+
+if [[ "$DESIRED_PYTHON"  == "3.8" ]]; then
+    NUMPY_VERSION="1.24.4"
+else
+    NUMPY_VERSION="1.26.0"
+fi
+conda install -y -c conda-forge numpy==${NUMPY_VERSION} pyyaml==6.0.1 patchelf==0.17.2 pygit2==1.13.2 openblas==0.3.24 ninja==1.11.1 scons==4.5.2
+
 python --version
 conda --version
diff --git a/build_m1_domains.sh b/build_m1_domains.sh
diff --git a/conda/pytorch-nightly/meta.yaml b/conda/pytorch-nightly/meta.yaml
@@ -21,17 +21,20 @@ requirements:
     - pyyaml
     {% if cross_compile_arm64 == 0 %}
     - mkl-include # [x86_64]
-    - mkl=2020.2 # [x86_64 and not win]
+    - mkl=2020.2 # [py <= 311 and x86_64 and not win]
+    - mkl=2023.1 # [py >= 312 and x86_64 and not win]
     - mkl=2021.4 # [x86_64 and win]
     {% endif %}
     - typing_extensions
     - ninja
     - libuv # [win]
     - numpy=1.19 # [py <= 39]
     - numpy=1.21.5 # [py == 310]
-    - numpy=1.23.5 # [py >= 311]
+    - numpy=1.23.5 # [py == 311]
+    - numpy=1.26.1 # [py >= 312]
     - openssl=1.1.1l # [py >= 38 and py <= 310 and linux]
-    - openssl=1.1.1s # [py >= 311 and linux]
+    - openssl=1.1.1s # [py == 311 and linux]
+    - openssl=3.1.4 # [py >= 312 and linux]
 {{ environ.get('PYTORCH_LLVM_PACKAGE', '    - llvmdev=9') }}
 {{ environ.get('MAGMA_PACKAGE', '') }}
 

diff --git a/conda/upload.sh b/conda/upload.sh
diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh
@@ -131,6 +131,9 @@ case ${DESIRED_PYTHON} in
   cp311*)
     retry pip install -q numpy==1.23.1
     ;;
+  cp312*)
+    retry pip install -q numpy==1.26.1
+    ;;
   # Should catch 3.9+
   *)
     retry pip install -q numpy==1.19.4

diff --git a/manywheel/build_cuda.sh b/manywheel/build_cuda.sh
@@ -47,7 +47,7 @@ if [[ -n "$DESIRED_CUDA" ]]; then
     # There really has to be a better way to do this - eli
     # Possibly limiting builds to specific cuda versions be delimiting images would be a choice
     if [[ "$OS_NAME" == *"Ubuntu"* ]]; then
-        echo "Switching to CUDA version $desired_cuda"
+        echo "Switching to CUDA version ${DESIRED_CUDA}"
         /builder/conda/switch_cuda_version.sh "${DESIRED_CUDA}"
     fi
 else
@@ -263,7 +263,9 @@ else
 fi
 
 # TODO: Remove me when Triton has a proper release channel
-if [[ $(uname) == "Linux" ]]; then
+# No triton dependency for now on 3.12 since we don't have binaries for it
+# and torch.compile doesn't work.
+if [[ $(uname) == "Linux" && "$DESIRED_PYTHON" != "3.12" ]]; then
     TRITON_SHORTHASH=$(cut -c1-10 $PYTORCH_ROOT/.github/ci_commit_pins/triton.txt)
 
     if [[ -z "$PYTORCH_EXTRA_INSTALL_REQUIREMENTS" ]]; then

diff --git a/manywheel/build_libtorch.sh b/manywheel/build_libtorch.sh
@@ -343,7 +343,7 @@ for pkg in /$LIBTORCH_HOUSE_DIR/libtorch*.zip; do
     fi
 
     # zip up the wheel back
-    zip -rq $(basename $pkg) $PREIX*
+    zip -rq $(basename $pkg) $PREFIX*
 
     # replace original wheel
     rm -f $pkg

diff --git a/manywheel/build_rocm.sh b/manywheel/build_rocm.sh
@@ -222,7 +222,9 @@ if [[ $ROCM_INT -ge 50600 ]]; then
 fi
 
 # Add triton install dependency
-if [[ $(uname) == "Linux" ]]; then
+# No triton dependency for now on 3.12 since we don't have binaries for it
+# and torch.compile doesn't work.
+if [[ $(uname) == "Linux" && "$DESIRED_PYTHON" != "3.12" ]]; then
     TRITON_SHORTHASH=$(cut -c1-10 $PYTORCH_ROOT/.ci/docker/ci_commit_pins/triton-rocm.txt)
     TRITON_VERSION=$(cat $PYTORCH_ROOT/.ci/docker/triton_version.txt)
 

diff --git a/manywheel/upload.sh b/manywheel/upload.sh
diff --git a/release/pypi/promote_pypi_to_production.sh b/release/pypi/promote_pypi_to_production.sh
@@ -34,3 +34,4 @@ promote_staging_binaries torch "${PYTORCH_VERSION}"
 promote_staging_binaries torchvision "${TORCHVISION_VERSION}"
 promote_staging_binaries torchaudio "${TORCHAUDIO_VERSION}"
 promote_staging_binaries torchtext "${TORCHTEXT_VERSION}"
+promote_staging_binaries torchdata "${TORCHDATA_VERSION}"
diff --git a/release/pypi/promote_pypi_to_staging.sh b/release/pypi/promote_pypi_to_staging.sh
@@ -23,26 +23,34 @@ upload_pypi_to_staging() {
 # Uncomment these to promote to pypi
 PYTORCH_LINUX_VERSION_SUFFIX="%2Bcu121.with.pypi.cudnn"
 LINUX_VERSION_SUFFIX="%2Bcu121"
-WIN_VERSION_SUFFIX="%2Bcpu"
+CPU_VERSION_SUFFIX="%2Bcpu"
 MACOS_X86_64="macosx_.*_x86_64"
 MACOS_ARM64="macosx_.*_arm64"
 
 PLATFORM="linux_x86_64"          VERSION_SUFFIX="${PYTORCH_LINUX_VERSION_SUFFIX}" upload_pypi_to_staging torch "${PYTORCH_VERSION}"
 PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX=""                                upload_pypi_to_staging torch "${PYTORCH_VERSION}"
-PLATFORM="win_amd64"             VERSION_SUFFIX="${WIN_VERSION_SUFFIX}"           upload_pypi_to_staging torch "${PYTORCH_VERSION}"
+PLATFORM="win_amd64"             VERSION_SUFFIX="${CPU_VERSION_SUFFIX}"           upload_pypi_to_staging torch "${PYTORCH_VERSION}"
 PLATFORM="${MACOS_X86_64}"       VERSION_SUFFIX=""                                upload_pypi_to_staging torch "${PYTORCH_VERSION}" # intel mac
 PLATFORM="${MACOS_ARM64}"        VERSION_SUFFIX=""                                upload_pypi_to_staging torch "${PYTORCH_VERSION}" # m1 mac
 
 PLATFORM="linux_x86_64"          VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}"
 PLATFORM="linux_aarch64"         VERSION_SUFFIX=""                        upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}"
-PLATFORM="win_amd64"             VERSION_SUFFIX="${WIN_VERSION_SUFFIX}"   upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}"
+PLATFORM="win_amd64"             VERSION_SUFFIX="${CPU_VERSION_SUFFIX}"   upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}"
 PLATFORM="${MACOS_X86_64}"       VERSION_SUFFIX=""                        upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}"
 PLATFORM="${MACOS_ARM64}"        VERSION_SUFFIX=""                        upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}"
 
 PLATFORM="linux_x86_64"          VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}"
 PLATFORM="linux_aarch64"         VERSION_SUFFIX=""                        upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}"
-PLATFORM="win_amd64"             VERSION_SUFFIX="${WIN_VERSION_SUFFIX}"   upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}"
+PLATFORM="win_amd64"             VERSION_SUFFIX="${CPU_VERSION_SUFFIX}"   upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}"
 PLATFORM="${MACOS_X86_64}"       VERSION_SUFFIX=""                        upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}"
 PLATFORM="${MACOS_ARM64}"        VERSION_SUFFIX=""                        upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}"
 
-upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}"
+PLATFORM="linux_x86" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}"
+PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}"
+PLATFORM="${MACOS_X86_64}" VERSION_SUFFIX="" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}"
+PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}"
+
+PLATFORM="manylinux2014_x86_64" VERSION_SUFFIX="" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}"
+PLATFORM="win_amd64" VERSION_SUFFIX="" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}"
+PLATFORM="${MACOS_X86_64}" VERSION_SUFFIX="" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}"
+PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}"
diff --git a/release/pypi/upload_pypi_to_staging.sh b/release/pypi/upload_pypi_to_staging.sh
@@ -43,7 +43,7 @@ fi
 for pkg in ${pkgs_to_promote}; do
     pkg_basename="$(basename "${pkg}")"
 
-    if [[ "${pkg}" != *aarch64* ]]; then
+    if [[ "${pkg}" != *aarch64* && "${pkg}" != *torchdata* ]]; then
         # sub out linux for manylinux1
         pkg_basename="$(basename "${pkg//linux/manylinux1}")"
     elif [[ "${pkg}" == *manylinux_2_17_aarch64* ]]; then

diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py
@@ -90,10 +90,12 @@ def check_version(package: str) -> None:
                 module_version = imported_module.__version__
                 if not module_version.startswith(release_matrix[module["name"]]):
                     raise RuntimeError(
-                        f"{module['name']} version mismatch, expected {release_matrix[module['name']]} for channel {channel}. But its {module_version}"
+                        f"{module['name']} version mismatch, expected: \
+                            {release_matrix[module['name']]} for channel {channel}. But its {module_version}"
                     )
                 else:
-                     print(f"{module['name']} version actual: {module_version} expected: {release_matrix[module['name']]} for channel {channel}.")
+                     print(f"{module['name']} version actual: {module_version} expected: \
+                        {release_matrix[module['name']]} for channel {channel}.")
 
     else:
         print(f"Skip version check for channel {channel} as stable version is None")

diff --git a/wheel/build_wheel.sh b/wheel/build_wheel.sh
@@ -136,23 +136,33 @@ export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
 SETUPTOOLS_PINNED_VERSION="=46.0.0"
 PYYAML_PINNED_VERSION="=5.3"
 EXTRA_CONDA_INSTALL_FLAGS=""
-case ${desired_python} in
+case $desired_python in
+    3.12)
+        echo "Using 3.12 deps"
+        SETUPTOOLS_PINNED_VERSION=">=68.0.0"
+        PYYAML_PINNED_VERSION=">=6.0.1"
+        NUMPY_PINNED_VERSION="==1.26.0"
+        ;;
     3.11)
+        echo "Using 3.11 deps"
         SETUPTOOLS_PINNED_VERSION=">=46.0.0"
         PYYAML_PINNED_VERSION=">=5.3"
         NUMPY_PINNED_VERSION="==1.23.5"
         ;;
     3.10)
+        echo "Using 3.10 deps"
         SETUPTOOLS_PINNED_VERSION=">=46.0.0"
         PYYAML_PINNED_VERSION=">=5.3"
         NUMPY_PINNED_VERSION="=1.21.2"
         ;;
     3.9)
+        echo "Using 3.9 deps"
         SETUPTOOLS_PINNED_VERSION=">=46.0.0"
         PYYAML_PINNED_VERSION=">=5.3"
         NUMPY_PINNED_VERSION="=1.19"
         ;;
     3.8)
+        echo "Using 3.8 deps"
         if [[ "$(uname -m)" == "arm64" ]]; then
           SETUPTOOLS_PINNED_VERSION=">=46.0.0"
           PYYAML_PINNED_VERSION=">=5.3"
@@ -162,6 +172,7 @@ case ${desired_python} in
         fi
         ;;
     *)
+        echo "Using default deps"
         NUMPY_PINNED_VERSION="=1.11.3"
         ;;
 esac