[ci][py3.11] Build python wheels on mac os for 3.11 (ray-project#36185)

Unifying the wheel building for macos since both could be done with conda. (We are already building wheels with conda for arm64) Added support for 3.11 wheels which is then trivial. (The system installed python seems to have 3.11 missing on buildkite still). Signed-off-by: Ricky Xu <[email protected]> Signed-off-by: e428265 <[email protected]>
lmco · Aug 31, 2023 · a27d817 · a27d817
1 parent 11ed130
commit a27d817
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 167 deletions.
diff --git a/ci/ci.sh b/ci/ci.sh
@@ -503,11 +503,7 @@ build_wheels_and_jars() {
       ;;
     darwin*)
       # This command should be kept in sync with ray/python/README-building-wheels.md.
-      if [ "$(uname -m)" = "arm64" ]; then
-        "${WORKSPACE_DIR}"/python/build-wheel-macos-arm64.sh
-      else
-        "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
-      fi
+      "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
       mkdir -p /tmp/artifacts/.whl
       rm -rf /tmp/artifacts/.whl || true
 

diff --git a/python/build-wheel-macos-arm64.sh b/python/build-wheel-macos-arm64.sh
diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh
@@ -6,79 +6,75 @@ set -e
 # Show explicitly which commands are currently running.
 set -x
 
-# Much of this is taken from https://github.com/matthew-brett/multibuild.
-# This script uses "sudo", so you may need to type in a password a couple times.
-
-MACPYTHON_URL=https://www.python.org/ftp/python
-MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
 DOWNLOAD_DIR=python_downloads
 
 NODE_VERSION="14"
-PY_VERSIONS=("3.7.0"
-             "3.8.2"
-             "3.9.1"
-             "3.10.4"
-             )
-PY_INSTS=("python-3.7.0-macosx10.6.pkg"
-          "python-3.8.2-macosx10.9.pkg"
-          "python-3.9.1-macosx10.9.pkg"
-          "python-3.10.4-macos11.pkg"
-          )
-PY_MMS=("3.7"
-        "3.8"
-        "3.9"
-        "3.10"
-        )
-
-NUMPY_VERSIONS=("1.14.5"
-                "1.14.5"
-                "1.19.3"
-                "1.22.0"
-                )
-
-./ci/env/install-bazel.sh
-
-mkdir -p $DOWNLOAD_DIR
-mkdir -p .whl
 
-# Use the latest version of Node.js in order to build the dashboard.
-source "$HOME"/.nvm/nvm.sh
-nvm install $NODE_VERSION
-nvm use $NODE_VERSION
+if [ "$(uname -m)" = "arm64" ]; then
+  # We Don't build wheels for Python 3.7 on Apple Silicon
+  PY_MMS=("3.8"
+          "3.9"
+          "3.10"
+          "3.11")
+else
+  PY_MMS=("3.7"
+          "3.8"
+          "3.9"
+          "3.10"
+          "3.11")
+fi
+
+if [[ -n "${SKIP_DEP_RES}" ]]; then
+  ./ci/env/install-bazel.sh
+
+  curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+
+  if [ "$(uname -m)" = "arm64" ]; then
+    curl -o- https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh | bash
+  else
+    curl -o- https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh | bash
+  fi
+
+  conda init bash
+  source ~/.bash_profile
+
+  # Use the latest version of Node.js in order to build the dashboard.
+  source "$HOME"/.nvm/nvm.sh
+  nvm install $NODE_VERSION
+  nvm use $NODE_VERSION
+fi
 
 # Build the dashboard so its static assets can be included in the wheel.
 pushd python/ray/dashboard/client
+  source "$HOME"/.nvm/nvm.sh
   npm ci
   npm run build
 popd
 
-for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
-  PY_VERSION=${PY_VERSIONS[i]}
-  PY_INST=${PY_INSTS[i]}
-  PY_MM=${PY_MMS[i]}
-  NUMPY_VERSION=${NUMPY_VERSIONS[i]}
+mkdir -p .whl
 
+for ((i=0; i<${#PY_MMS[@]}; ++i)); do
+  PY_MM=${PY_MMS[i]}
+  CONDA_ENV_NAME="p$PY_MM"
+
   # The -f flag is passed twice to also run git clean in the arrow subdirectory.
   # The -d flag removes directories. The -x flag ignores the .gitignore file,
   # and the -e flag ensures that we don't remove the .whl directory.
   git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR -e python/ray/dashboard/client -e dashboard/client
 
-  # Install Python.
-  # In Buildkite, the Python packages are installed on the machien before the build has ran.
-  PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM
-  PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM"
-
-  if [ -z "${BUILDKITE}" ]; then
-    INST_PATH=python_downloads/$PY_INST
-    curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH"
-    sudo installer -pkg "$INST_PATH" -target /
-    installer -pkg "$INST_PATH" -target /
-
-    pushd /tmp
-      # Install latest version of pip to avoid brownouts.
-      curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE
-    popd
-  fi
+  # Install python using conda. This should be easier to produce consistent results in buildkite and locally.
+  [ ! -f "$HOME/.bash_profile" ] && conda init bash
+  source ~/.bash_profile
+  conda create -y -n "$CONDA_ENV_NAME"
+  conda activate "$CONDA_ENV_NAME"
+  conda remove -y python || true
+  conda install -y python="$PY_MM"
+
+  # NOTE: We expect conda to set the PATH properly.
+  PIP_CMD=pip
+  PYTHON_EXE=python
+
+  $PIP_CMD install --upgrade pip
 
   if [ -z "${TRAVIS_COMMIT}" ]; then
     TRAVIS_COMMIT=${BUILDKITE_COMMIT}
@@ -88,27 +84,25 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
     # Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade.
     # TODO: Unpin after https://github.com/pypa/setuptools/issues/2849 is fixed.
     $PIP_CMD install --upgrade setuptools==58.4
-    # Install setuptools_scm because otherwise when building the wheel for
-    # Python 3.6, we see an error.
-    $PIP_CMD install -q setuptools_scm==3.1.0
-    # Fix the numpy version because this will be the oldest numpy version we can
-    # support.
-    $PIP_CMD install -q numpy=="$NUMPY_VERSION" cython==0.29.32
+    $PIP_CMD install -q cython==0.29.32
     # Install wheel to avoid the error "invalid command 'bdist_wheel'".
     $PIP_CMD install -q wheel
     # Set the commit SHA in __init__.py.
     if [ -n "$TRAVIS_COMMIT" ]; then
-      sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak
+      echo "TRAVIS_COMMIT variable detected. ray.__commit__ will be set to $TRAVIS_COMMIT"
     else
-      echo "TRAVIS_COMMIT variable not set - required to populated ray.__commit__."
-      exit 1
+      echo "TRAVIS_COMMIT variable is not set, getting the current commit from git."
+      TRAVIS_COMMIT=$(git rev-parse HEAD)
     fi
+
+    sed -i .bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak
+
     # Add the correct Python to the path and build the wheel. This is only
     # needed so that the installation finds the cython executable.
     # build ray wheel
-    PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel
+    $PYTHON_EXE setup.py bdist_wheel
     # build ray-cpp wheel
-    RAY_INSTALL_CPP=1 PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel
+    RAY_INSTALL_CPP=1 $PYTHON_EXE setup.py bdist_wheel
     mv dist/*.whl ../.whl/
   popd
 done