From 3b74d53e411880296200d7bfb3e1af08fc06c2b4 Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Tue, 6 Jun 2023 13:47:16 +0800 Subject: [PATCH 01/10] init Signed-off-by: Ricky Xu --- python/build-wheel-macos.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 2596cc452910..b031c2c8769b 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -18,16 +18,19 @@ PY_VERSIONS=("3.7.0" "3.8.2" "3.9.1" "3.10.4" + "3.11.1" ) PY_INSTS=("python-3.7.0-macosx10.6.pkg" "python-3.8.2-macosx10.9.pkg" "python-3.9.1-macosx10.9.pkg" "python-3.10.4-macos11.pkg" + "python-3.11.1-macos11.pkg" ) PY_MMS=("3.7" "3.8" "3.9" "3.10" + "3.11" ) NUMPY_VERSIONS=("1.14.5" From 3d1d332d873fa4fc4f8cce94ea85c175738cc549 Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Tue, 6 Jun 2023 14:18:20 +0800 Subject: [PATCH 02/10] update Signed-off-by: Ricky Xu --- python/build-wheel-macos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index b031c2c8769b..02d708b77930 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -37,6 +37,7 @@ NUMPY_VERSIONS=("1.14.5" "1.14.5" "1.19.3" "1.22.0" + "1.24.3" ) ./ci/env/install-bazel.sh From 418963c6412fef01437b33a8b79bd0a9629ef485 Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Tue, 6 Jun 2023 15:02:27 +0800 Subject: [PATCH 03/10] add arm64 Signed-off-by: Ricky Xu --- python/build-wheel-macos-arm64.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/build-wheel-macos-arm64.sh b/python/build-wheel-macos-arm64.sh index 9635828adf87..6c3d257d4c7a 100755 --- a/python/build-wheel-macos-arm64.sh +++ b/python/build-wheel-macos-arm64.sh @@ -11,10 +11,12 @@ DOWNLOAD_DIR=python_downloads NODE_VERSION="14" PY_VERSIONS=("3.8.2" "3.9.1" - "3.10.4") + "3.10.4" + "3.11.1") PY_MMS=("3.8" "3.9" - "3.10") + "3.10" + "3.11") if [[ -n "${SKIP_DEP_RES}" ]]; then From 1105af45f7d9de4318a1e8d5585e72c588bc4152 Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Tue, 6 Jun 2023 23:23:32 +0800 Subject: [PATCH 04/10] try Signed-off-by: Ricky Xu --- python/build-wheel-macos.sh | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 02d708b77930..8f229606a05d 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -72,17 +72,15 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM" - if [ -z "${BUILDKITE}" ]; then - INST_PATH=python_downloads/$PY_INST - curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH" - sudo installer -pkg "$INST_PATH" -target / - installer -pkg "$INST_PATH" -target / - - pushd /tmp - # Install latest version of pip to avoid brownouts. - curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE - popd - fi + INST_PATH=python_downloads/$PY_INST + curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH" + sudo installer -pkg "$INST_PATH" -target / + installer -pkg "$INST_PATH" -target / + + pushd /tmp + # Install latest version of pip to avoid brownouts. + curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE + popd if [ -z "${TRAVIS_COMMIT}" ]; then TRAVIS_COMMIT=${BUILDKITE_COMMIT} From d9df79bf123ccc9af096914a7142ab0034b92876 Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Thu, 8 Jun 2023 13:27:28 +0800 Subject: [PATCH 05/10] try Signed-off-by: Ricky Xu --- ci/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/ci.sh b/ci/ci.sh index e8468bd63a5a..9386cc2d5929 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -506,7 +506,7 @@ build_wheels_and_jars() { if [ "$(uname -m)" = "arm64" ]; then "${WORKSPACE_DIR}"/python/build-wheel-macos-arm64.sh else - "${WORKSPACE_DIR}"/python/build-wheel-macos.sh + "${WORKSPACE_DIR}"/python/build-wheel-macos-arm64.sh fi mkdir -p /tmp/artifacts/.whl rm -rf /tmp/artifacts/.whl || true From a47d2af44925774b36579dddf61982157d23eaf6 Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Thu, 8 Jun 2023 13:59:38 +0800 Subject: [PATCH 06/10] update Signed-off-by: Ricky Xu --- ci/ci.sh | 6 +----- python/build-wheel-macos-arm64.sh | 16 +++++++++------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ci/ci.sh b/ci/ci.sh index 9386cc2d5929..8ea25cb8280b 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -503,11 +503,7 @@ build_wheels_and_jars() { ;; darwin*) # This command should be kept in sync with ray/python/README-building-wheels.md. - if [ "$(uname -m)" = "arm64" ]; then - "${WORKSPACE_DIR}"/python/build-wheel-macos-arm64.sh - else - "${WORKSPACE_DIR}"/python/build-wheel-macos-arm64.sh - fi + "${WORKSPACE_DIR}"/python/build-wheel-macos-arm64.sh mkdir -p /tmp/artifacts/.whl rm -rf /tmp/artifacts/.whl || true diff --git a/python/build-wheel-macos-arm64.sh b/python/build-wheel-macos-arm64.sh index 6c3d257d4c7a..6db7b1f9f103 100755 --- a/python/build-wheel-macos-arm64.sh +++ b/python/build-wheel-macos-arm64.sh @@ -9,11 +9,8 @@ set -x DOWNLOAD_DIR=python_downloads NODE_VERSION="14" -PY_VERSIONS=("3.8.2" - "3.9.1" - "3.10.4" - "3.11.1") -PY_MMS=("3.8" +PY_MMS=("3.7" + "3.8" "3.9" "3.10" "3.11") @@ -23,7 +20,12 @@ if [[ -n "${SKIP_DEP_RES}" ]]; then ./ci/env/install-bazel.sh curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash - curl -o- https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh | bash + + if [ "$(uname -m)" = "arm64" ]; then + curl -o- https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh | bash + else + curl -o- https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh | bash + conda init bash source ~/.bash_profile @@ -42,7 +44,7 @@ popd mkdir -p .whl -for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do +for ((i=0; i<${#PY_MMS[@]}; ++i)); do PY_MM=${PY_MMS[i]} CONDA_ENV_NAME="p$PY_MM" From 1fa0e7a8caae454c0c696eb635de1e0471d80459 Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Thu, 8 Jun 2023 14:00:00 +0800 Subject: [PATCH 07/10] remove Signed-off-by: Ricky Xu --- python/build-wheel-macos.sh | 116 ------------------------------------ 1 file changed, 116 deletions(-) delete mode 100755 python/build-wheel-macos.sh diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh deleted file mode 100755 index 8f229606a05d..000000000000 --- a/python/build-wheel-macos.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -# Cause the script to exit if a single command fails. -set -e - -# Show explicitly which commands are currently running. -set -x - -# Much of this is taken from https://github.com/matthew-brett/multibuild. -# This script uses "sudo", so you may need to type in a password a couple times. - -MACPYTHON_URL=https://www.python.org/ftp/python -MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions -DOWNLOAD_DIR=python_downloads - -NODE_VERSION="14" -PY_VERSIONS=("3.7.0" - "3.8.2" - "3.9.1" - "3.10.4" - "3.11.1" - ) -PY_INSTS=("python-3.7.0-macosx10.6.pkg" - "python-3.8.2-macosx10.9.pkg" - "python-3.9.1-macosx10.9.pkg" - "python-3.10.4-macos11.pkg" - "python-3.11.1-macos11.pkg" - ) -PY_MMS=("3.7" - "3.8" - "3.9" - "3.10" - "3.11" - ) - -NUMPY_VERSIONS=("1.14.5" - "1.14.5" - "1.19.3" - "1.22.0" - "1.24.3" - ) - -./ci/env/install-bazel.sh - -mkdir -p $DOWNLOAD_DIR -mkdir -p .whl - -# Use the latest version of Node.js in order to build the dashboard. -source "$HOME"/.nvm/nvm.sh -nvm install $NODE_VERSION -nvm use $NODE_VERSION - -# Build the dashboard so its static assets can be included in the wheel. -pushd python/ray/dashboard/client - npm ci - npm run build -popd - -for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do - PY_VERSION=${PY_VERSIONS[i]} - PY_INST=${PY_INSTS[i]} - PY_MM=${PY_MMS[i]} - NUMPY_VERSION=${NUMPY_VERSIONS[i]} - - # The -f flag is passed twice to also run git clean in the arrow subdirectory. - # The -d flag removes directories. The -x flag ignores the .gitignore file, - # and the -e flag ensures that we don't remove the .whl directory. - git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR -e python/ray/dashboard/client -e dashboard/client - - # Install Python. - # In Buildkite, the Python packages are installed on the machien before the build has ran. - PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM - PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM" - - INST_PATH=python_downloads/$PY_INST - curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH" - sudo installer -pkg "$INST_PATH" -target / - installer -pkg "$INST_PATH" -target / - - pushd /tmp - # Install latest version of pip to avoid brownouts. - curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE - popd - - if [ -z "${TRAVIS_COMMIT}" ]; then - TRAVIS_COMMIT=${BUILDKITE_COMMIT} - fi - - pushd python - # Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade. - # TODO: Unpin after https://github.com/pypa/setuptools/issues/2849 is fixed. - $PIP_CMD install --upgrade setuptools==58.4 - # Install setuptools_scm because otherwise when building the wheel for - # Python 3.6, we see an error. - $PIP_CMD install -q setuptools_scm==3.1.0 - # Fix the numpy version because this will be the oldest numpy version we can - # support. - $PIP_CMD install -q numpy=="$NUMPY_VERSION" cython==0.29.32 - # Install wheel to avoid the error "invalid command 'bdist_wheel'". - $PIP_CMD install -q wheel - # Set the commit SHA in __init__.py. - if [ -n "$TRAVIS_COMMIT" ]; then - sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak - else - echo "TRAVIS_COMMIT variable not set - required to populated ray.__commit__." - exit 1 - fi - # Add the correct Python to the path and build the wheel. This is only - # needed so that the installation finds the cython executable. - # build ray wheel - PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel - # build ray-cpp wheel - RAY_INSTALL_CPP=1 PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel - mv dist/*.whl ../.whl/ - popd -done From 2e8c52588310c22c017ca811a6eb801bb0ccbe96 Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Thu, 8 Jun 2023 14:00:09 +0800 Subject: [PATCH 08/10] done Signed-off-by: Ricky Xu --- python/{build-wheel-macos-arm64.sh => build-wheel-macos.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename python/{build-wheel-macos-arm64.sh => build-wheel-macos.sh} (100%) diff --git a/python/build-wheel-macos-arm64.sh b/python/build-wheel-macos.sh similarity index 100% rename from python/build-wheel-macos-arm64.sh rename to python/build-wheel-macos.sh From c60489ce4d37b4341015ee87276ea109504d2a4f Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Thu, 8 Jun 2023 16:02:55 +0800 Subject: [PATCH 09/10] update Signed-off-by: Ricky Xu --- python/build-wheel-macos.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 6db7b1f9f103..42dfd829c31a 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -9,12 +9,20 @@ set -x DOWNLOAD_DIR=python_downloads NODE_VERSION="14" -PY_MMS=("3.7" - "3.8" - "3.9" - "3.10" - "3.11") +if [ "$(uname -m)" = "arm64" ]; then + # We Don't build wheels for Python 3.7 on Apple Silicon + PY_MMS=("3.8" + "3.9" + "3.10" + "3.11") +else + PY_MMS=("3.7" + "3.8" + "3.9" + "3.10" + "3.11") +fi if [[ -n "${SKIP_DEP_RES}" ]]; then ./ci/env/install-bazel.sh @@ -25,6 +33,7 @@ if [[ -n "${SKIP_DEP_RES}" ]]; then curl -o- https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh | bash else curl -o- https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh | bash + fi conda init bash source ~/.bash_profile From 098160575b7e506f56576a235f89643a5f910380 Mon Sep 17 00:00:00 2001 From: Ricky Xu Date: Thu, 8 Jun 2023 21:38:36 +0800 Subject: [PATCH 10/10] hm Signed-off-by: Ricky Xu --- ci/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/ci.sh b/ci/ci.sh index 8ea25cb8280b..bf7d57156ccb 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -503,7 +503,7 @@ build_wheels_and_jars() { ;; darwin*) # This command should be kept in sync with ray/python/README-building-wheels.md. - "${WORKSPACE_DIR}"/python/build-wheel-macos-arm64.sh + "${WORKSPACE_DIR}"/python/build-wheel-macos.sh mkdir -p /tmp/artifacts/.whl rm -rf /tmp/artifacts/.whl || true