[ci][py3.11] Build python wheels on mac os for 3.11

ci/ci.sh

@@ -503,11 +503,7 @@ build_wheels_and_jars() {
       ;;
     darwin*)
       # This command should be kept in sync with ray/python/README-building-wheels.md.
-      if [ "$(uname -m)" = "arm64" ]; then
-        "${WORKSPACE_DIR}"/python/build-wheel-macos-arm64.sh
-      else
-        "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
-      fi
+      "${WORKSPACE_DIR}"/python/build-wheel-macos-arm64.sh
       mkdir -p /tmp/artifacts/.whl
       rm -rf /tmp/artifacts/.whl || true
 

python/build-wheel-macos.sh

@@ -6,79 +6,66 @@ set -e
 # Show explicitly which commands are currently running.
 set -x
 
-# Much of this is taken from https://github.com/matthew-brett/multibuild.
-# This script uses "sudo", so you may need to type in a password a couple times.
-
-MACPYTHON_URL=https://www.python.org/ftp/python
-MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
 DOWNLOAD_DIR=python_downloads
 
 NODE_VERSION="14"
-PY_VERSIONS=("3.7.0"
-             "3.8.2"
-             "3.9.1"
-             "3.10.4"
-             )
-PY_INSTS=("python-3.7.0-macosx10.6.pkg"
-          "python-3.8.2-macosx10.9.pkg"
-          "python-3.9.1-macosx10.9.pkg"
-          "python-3.10.4-macos11.pkg"
-          )
 PY_MMS=("3.7"
         "3.8"
         "3.9"
         "3.10"
-        )
+        "3.11")
 
-NUMPY_VERSIONS=("1.14.5"
-                "1.14.5"
-                "1.19.3"
-                "1.22.0"
-                )
 
-./ci/env/install-bazel.sh
+if [[ -n "${SKIP_DEP_RES}" ]]; then
+  ./ci/env/install-bazel.sh
 
-mkdir -p $DOWNLOAD_DIR
-mkdir -p .whl
+  curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+
+  if [ "$(uname -m)" = "arm64" ]; then
+    curl -o- https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh | bash
+  else
+    curl -o- https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh | bash
 
-# Use the latest version of Node.js in order to build the dashboard.
-source "$HOME"/.nvm/nvm.sh
-nvm install $NODE_VERSION
-nvm use $NODE_VERSION
+  conda init bash
+  source ~/.bash_profile
+
+  # Use the latest version of Node.js in order to build the dashboard.
+  source "$HOME"/.nvm/nvm.sh
+  nvm install $NODE_VERSION
+  nvm use $NODE_VERSION
+fi
 
 # Build the dashboard so its static assets can be included in the wheel.
 pushd python/ray/dashboard/client
+  source "$HOME"/.nvm/nvm.sh
   npm ci
   npm run build
 popd
 
-for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
-  PY_VERSION=${PY_VERSIONS[i]}
-  PY_INST=${PY_INSTS[i]}
-  PY_MM=${PY_MMS[i]}
-  NUMPY_VERSION=${NUMPY_VERSIONS[i]}
+mkdir -p .whl
 
+for ((i=0; i<${#PY_MMS[@]}; ++i)); do
+  PY_MM=${PY_MMS[i]}
+  CONDA_ENV_NAME="p$PY_MM"
+
   # The -f flag is passed twice to also run git clean in the arrow subdirectory.
   # The -d flag removes directories. The -x flag ignores the .gitignore file,
   # and the -e flag ensures that we don't remove the .whl directory.
   git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR -e python/ray/dashboard/client -e dashboard/client
 
-  # Install Python.
-  # In Buildkite, the Python packages are installed on the machien before the build has ran.
-  PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM
-  PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM"
-
-  if [ -z "${BUILDKITE}" ]; then
-    INST_PATH=python_downloads/$PY_INST
-    curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH"
-    sudo installer -pkg "$INST_PATH" -target /
-    installer -pkg "$INST_PATH" -target /
-
-    pushd /tmp
-      # Install latest version of pip to avoid brownouts.
-      curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE
-    popd
-  fi
+  # Install python using conda. This should be easier to produce consistent results in buildkite and locally.
+  [ ! -f "$HOME/.bash_profile" ] && conda init bash
+  source ~/.bash_profile
+  conda create -y -n "$CONDA_ENV_NAME"
+  conda activate "$CONDA_ENV_NAME"
+  conda remove -y python || true
+  conda install -y python="$PY_MM"
+
+  # NOTE: We expect conda to set the PATH properly.
+  PIP_CMD=pip
+  PYTHON_EXE=python
+
+  $PIP_CMD install --upgrade pip
 
   if [ -z "${TRAVIS_COMMIT}" ]; then
     TRAVIS_COMMIT=${BUILDKITE_COMMIT}
@@ -88,27 +75,25 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
     # Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade.
     # TODO: Unpin after https://github.com/pypa/setuptools/issues/2849 is fixed.
     $PIP_CMD install --upgrade setuptools==58.4
-    # Install setuptools_scm because otherwise when building the wheel for
-    # Python 3.6, we see an error.
-    $PIP_CMD install -q setuptools_scm==3.1.0
-    # Fix the numpy version because this will be the oldest numpy version we can
-    # support.
-    $PIP_CMD install -q numpy=="$NUMPY_VERSION" cython==0.29.32
+    $PIP_CMD install -q cython==0.29.32
     # Install wheel to avoid the error "invalid command 'bdist_wheel'".
     $PIP_CMD install -q wheel
     # Set the commit SHA in __init__.py.
     if [ -n "$TRAVIS_COMMIT" ]; then
-      sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak
+      echo "TRAVIS_COMMIT variable detected. ray.__commit__ will be set to $TRAVIS_COMMIT"
     else
-      echo "TRAVIS_COMMIT variable not set - required to populated ray.__commit__."
-      exit 1
+      echo "TRAVIS_COMMIT variable is not set, getting the current commit from git."
+      TRAVIS_COMMIT=$(git rev-parse HEAD)
     fi
+
+    sed -i .bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak
+
     # Add the correct Python to the path and build the wheel. This is only
     # needed so that the installation finds the cython executable.
     # build ray wheel
-    PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel
+    $PYTHON_EXE setup.py bdist_wheel
     # build ray-cpp wheel
-    RAY_INSTALL_CPP=1 PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel
+    RAY_INSTALL_CPP=1 $PYTHON_EXE setup.py bdist_wheel
     mv dist/*.whl ../.whl/
   popd
 done