From 9cfbff0772a5005a0c96ad79a0bc18b8582a7d65 Mon Sep 17 00:00:00 2001 From: mantaionut Date: Sun, 15 Oct 2023 16:30:31 +0300 Subject: [PATCH] Use intel Sparse on Windows Use Sparse on Windows and updated mkl to 2021.4.0 --- conda/pytorch-nightly/bld.bat | 5 ----- conda/pytorch-nightly/meta.yaml | 4 +++- wheel/build_wheel.sh | 2 +- windows/build_pytorch.bat | 6 +----- windows/internal/copy.bat | 4 ++++ windows/internal/copy_cpu.bat | 6 ++++++ windows/internal/smoke_test.bat | 19 +++++++++++-------- 7 files changed, 26 insertions(+), 20 deletions(-) diff --git a/conda/pytorch-nightly/bld.bat b/conda/pytorch-nightly/bld.bat index 972df7e9cf..775256ea7f 100644 --- a/conda/pytorch-nightly/bld.bat +++ b/conda/pytorch-nightly/bld.bat @@ -34,11 +34,6 @@ if "%desired_cuda%" == "12.1" ( set DISTUTILS_USE_SDK=1 -curl https://s3.amazonaws.com/ossci-windows/mkl_2020.2.254.7z -k -O -7z x -aoa mkl_2020.2.254.7z -omkl -set CMAKE_INCLUDE_PATH=%SRC_DIR%\mkl\include -set LIB=%SRC_DIR%\mkl\lib;%LIB% - set libuv_ROOT=%PREFIX%\Library echo libuv_ROOT=%libuv_ROOT% diff --git a/conda/pytorch-nightly/meta.yaml b/conda/pytorch-nightly/meta.yaml index 79025dc8c1..767d19812c 100644 --- a/conda/pytorch-nightly/meta.yaml +++ b/conda/pytorch-nightly/meta.yaml @@ -19,6 +19,7 @@ requirements: - python - setuptools - pyyaml + - mkl-devel=2021.4.0 # [win] {% if cross_compile_arm64 == 0 %} - mkl-include # [x86_64] - mkl=2020.2 # [x86_64 and not win] @@ -37,8 +38,9 @@ requirements: run: - python + - mkl==2021.4.0 # [win] {% if cross_compile_arm64 == 0 %} - - mkl >=2018 # [x86_64] + - mkl==2021.4 # [x86_64] {% endif %} - libuv # [win] - intel-openmp # [win] diff --git a/wheel/build_wheel.sh b/wheel/build_wheel.sh index 75138fc094..b29c066ed1 100755 --- a/wheel/build_wheel.sh +++ b/wheel/build_wheel.sh @@ -177,7 +177,7 @@ fi if [[ "$(uname -m)" == "arm64" ]]; then retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq cmake ninja else - retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq cmake ninja mkl-include==2022.2.1 mkl-static==2022.2.1 -c intel + retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq cmake ninja mkl-include==2022.2.1 mkl-static==2022.2.1 mkl==2021.4.0 -c intel fi retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true diff --git a/windows/build_pytorch.bat b/windows/build_pytorch.bat index 6351387357..33bb29419b 100644 --- a/windows/build_pytorch.bat +++ b/windows/build_pytorch.bat @@ -67,10 +67,6 @@ exit /B 1 :: Install MKL rmdir /s /q mkl del mkl_2020.2.254.7z -curl https://s3.amazonaws.com/ossci-windows/mkl_2020.2.254.7z -k -O -7z x -aoa mkl_2020.2.254.7z -omkl -set CMAKE_INCLUDE_PATH=%cd%\mkl\include -set LIB=%cd%\mkl\lib;%LIB% :: Download MAGMA Files on CUDA builds set MAGMA_VERSION=2.5.4 @@ -128,7 +124,7 @@ for %%v in (%DESIRED_PYTHON_PREFIX%) do ( ) else ( set "PATH=%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%" ) - pip install ninja + pip install ninja mkl-include==2021.4.0 mkl-devel==2021.4.0 @setlocal :: Set Flags if not "%CUDA_VERSION%"=="cpu" ( diff --git a/windows/internal/copy.bat b/windows/internal/copy.bat index 490d9593a1..9893fc7c52 100755 --- a/windows/internal/copy.bat +++ b/windows/internal/copy.bat @@ -11,6 +11,10 @@ copy "%CUDA_PATH%\extras\CUPTI\lib64\cupti64_*.dll*" pytorch\torch\lib copy "C:\Program Files\NVIDIA Corporation\NvToolsExt\bin\x64\nvToolsExt64_1.dll*" pytorch\torch\lib copy "%CONDA_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib +IF "%PACKAGE_TYPE%"=="libtorch" ( + copy "%CONDA_LIB_PATH%\mkl_intel_thread.1.dll" pytorch\torch\lib + copy "%CONDA_LIB_PATH%\mkl_core.1.dll" pytorch\torch\lib +) :: Should be set in build_pytorch.bat copy "%libuv_ROOT%\bin\uv.dll" pytorch\torch\lib diff --git a/windows/internal/copy_cpu.bat b/windows/internal/copy_cpu.bat index 2dae4613ee..e089c59512 100755 --- a/windows/internal/copy_cpu.bat +++ b/windows/internal/copy_cpu.bat @@ -1,3 +1,9 @@ +@echo on copy "%CONDA_LIB_PATH%\libiomp*5md.dll" pytorch\torch\lib :: Should be set in build_pytorch.bat copy "%libuv_ROOT%\bin\uv.dll" pytorch\torch\lib + +IF "%PACKAGE_TYPE%"=="libtorch" ( + copy "%CONDA_LIB_PATH%\mkl_intel_thread.1.dll" pytorch\torch\lib + copy "%CONDA_LIB_PATH%\mkl_core.1.dll" pytorch\torch\lib +) \ No newline at end of file diff --git a/windows/internal/smoke_test.bat b/windows/internal/smoke_test.bat index ad276b9928..9e06dd285f 100644 --- a/windows/internal/smoke_test.bat +++ b/windows/internal/smoke_test.bat @@ -53,7 +53,7 @@ if errorlevel 1 exit /b 1 set "PATH=%CD%\Python%PYTHON_VERSION%\Scripts;%CD%\Python;%PATH%" -pip install -q numpy protobuf "mkl>=2019" +pip install -q numpy protobuf if errorlevel 1 exit /b 1 for /F "delims=" %%i in ('where /R "%PYTORCH_FINAL_PACKAGE_DIR:/=\%" *.whl') do pip install "%%i" @@ -86,14 +86,18 @@ set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%" conda create -qyn testenv python=%DESIRED_PYTHON% if errorlevel 1 exit /b 1 - +call conda install -yq conda-build +if errorlevel 1 exit /b 1 call %CONDA_HOME%\condabin\activate.bat testenv if errorlevel 1 exit /b 1 +set "NO_ARCH_PATH=%PYTORCH_FINAL_PACKAGE_DIR:/=\%\noarch" +mkdir %NO_ARCH_PATH% +for /F "delims=" %%i in ('where /R "%PYTORCH_FINAL_PACKAGE_DIR:/=\%" *.tar.bz2') do xcopy "%%i" %NO_ARCH_PATH% /Y +if ERRORLEVEL 1 exit /b 1 +call conda index %PYTORCH_FINAL_PACKAGE_DIR% +if errorlevel 1 exit /b 1 +call conda install -yq -c "file:///%PYTORCH_FINAL_PACKAGE_DIR%" pytorch==%PYTORCH_BUILD_VERSION% -c pytorch -c numba/label/dev -c pytorch-nightly -c nvidia -:: do conda install to make sure all the dependencies are installed -:: Install numpy see: https://github.com/pytorch/pytorch/issues/107228 -:: todo: Remove numpy install once the issue above is resolved -call conda install -yq numpy pytorch %CONDA_EXTRA_ARGS% if ERRORLEVEL 1 exit /b 1 set /a CUDA_VER=%CUDA_VERSION% @@ -102,8 +106,7 @@ set CUDA_VER_MINOR=%CUDA_VERSION:~-1,1% set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR% :: Install package we just build -for /F "delims=" %%i in ('where /R "%PYTORCH_FINAL_PACKAGE_DIR:/=\%" *.tar.bz2') do call conda install -yq "%%i" --offline -if ERRORLEVEL 1 exit /b 1 + :smoke_test python -c "import torch"