Prepare Python 3.11 support (#31170)

Preliminary PR for adding Python 3.11 support, mapping out various dependencies, fixing issues. ### Main changes - Upgrade cython to 0.29.32 - Add CI/CD steps for 3.11 wheel - Change cython code to not use `recursion_depth` - Update cloudpickle to latest - Use newer manylinux2014 which has python3.11 - Condition certain python packages in requirements.txt on <3.11 that don't yet have a 3.11 version ### Checklist: - cython - [x] remove deprecated `recursion_depth` - [x] exc_type cython/cython#4500 - [ ] package dependencies https://pyreadiness.org/3.11/ - [ ] llvmlite numba/llvmlite#869 - [ ] numba numba/numba#8304 - [ ] pyarrow - [ ] scikit-learn - [ ] pydantic - [x] cloudpickle - [x] upgrade cython to 0.29.32 - [ ] tensorflow tensorflow/tensorflow#58032 - [ ] torch pytorch/pytorch#86566 - [ ] miniconda conda/conda#11170 - [ ] `docker/base-deps/Dockerfile` - [x] claim to support 3.11 in setup.py - [ ] cicd - [ ] .buildkite/ - [ ] .buildkite/pipeline.build.yml - [ ] ci/ - [ ] ci/build/test-wheels.sh - [ ] ci/build/build-docker-images.py - [ ] release tests - [ ] docker/retag-lambda/python_versions.txt - [ ] download_wheels.sh - [ ] wheels - [ ] `python/build-wheel-macos.sh` - [ ] `python/build-wheel-windows.sh` - [ ] Tests - [ ] pytest ray/serve/tests - [ ] python python/ray/serve/examples/echo_full.py - [ ] bazel test //:core_worker_test - [ ] bazel test --build_tests_only //:all - [ ] //python/ray/tests:test_pydantic_serialization fastapi/fastapi#5048 - [ ] //python/ray/train:test_torch_utils - [ ] Documentation - [x] installation.rst Current status: Linux and mac wheels build in CICD. Docker images will come in a separate PR.
ray-project · Jan 12, 2023 · 4ddc7c3 · 4ddc7c3
1 parent 4a00526
commit 4ddc7c3
Show file tree

Hide file tree

Showing 21 changed files with 368 additions and 235 deletions.
diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
@@ -337,6 +337,14 @@
     - ./ci/ci.sh test_minimal 3.10
     - ./ci/ci.sh test_latest_core_dependencies 3.10
 
+- label: ":python: Minimal install 3.11"
+  conditions: ["RAY_CI_PYTHON_AFFECTED"]
+  instance_size: medium
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/ci.sh test_minimal 3.11
+    - ./ci/ci.sh test_latest_core_dependencies 3.11
+
 - label: ":python: Default install"
   conditions: ["RAY_CI_PYTHON_AFFECTED"]
   instance_size: small

diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl
@@ -194,8 +194,8 @@ def ray_deps_setup():
     auto_http_archive(
         name = "cython",
         build_file = True,
-        url = "https://github.com/cython/cython/archive/3028e8c7ac296bc848d996e397c3354b3dbbd431.tar.gz",
-        sha256 = "31ea23c2231ddee8572a2a5effd54952e16a1b44e9a4cb3eb645418f8accf20d",
+        url = "https://github.com/cython/cython/archive/c48361d0a0969206e227ec016f654c9d941c2b69.tar.gz",
+        sha256 = "37c466fea398da9785bc37fe16f1455d2645d21a72e402103991d9e2fa1c6ff3",
     )
 
     auto_http_archive(

diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py
@@ -620,7 +620,7 @@ def push_readmes(merge_build: bool):
         default="py37",
         nargs="*",
         help="Which python versions to build. "
-        "Must be in (py36, py37, py38, py39, py310)",
+        "Must be in (py36, py37, py38, py39, py310, py311)",
     )
     parser.add_argument(
         "--device-types",

diff --git a/ci/build/test-wheels.sh b/ci/build/test-wheels.sh
@@ -100,7 +100,8 @@ elif [[ "$platform" == "macosx" ]]; then
           "3.7"
           "3.8"
           "3.9"
-          "3.10")
+          "3.10"
+          )
 
   for ((i=0; i<${#PY_MMS[@]}; ++i)); do
     PY_MM="${PY_MMS[i]}"

diff --git a/ci/ci.sh b/ci/ci.sh
@@ -414,8 +414,7 @@ validate_wheels_commit_str() {
       continue
     fi
 
-    folder=${basename%%-cp*}
-    WHL_COMMIT=$(unzip -p "$whl" "${folder}.data/purelib/ray/__init__.py" | grep "__commit__" | awk -F'"' '{print $2}')
+    WHL_COMMIT=$(unzip -p "$whl" | grep "^__commit__" | awk -F'"' '{print $2}')
 
     if [ "${WHL_COMMIT}" != "${EXPECTED_COMMIT}" ]; then
       echo "Error: Observed wheel commit (${WHL_COMMIT}) is not expected commit (${EXPECTED_COMMIT}). Aborting."
@@ -459,7 +458,7 @@ build_wheels() {
         # This command should be kept in sync with ray/python/README-building-wheels.md,
         # except the "${MOUNT_BAZEL_CACHE[@]}" part.
         docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
-        quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh
+        quay.io/pypa/manylinux2014_x86_64:2022-12-20-b4884d9 /ray/python/build-wheel-manylinux2014.sh
       else
         rm -rf /ray-mount/*
         rm -rf /ray-mount/.whl || true
@@ -469,7 +468,7 @@ build_wheels() {
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /ray-mounted
         docker run --rm -w /ray -v /ray:/ray "${MOUNT_BAZEL_CACHE[@]}" \
-          quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh
+          quay.io/pypa/manylinux2014_x86_64:2022-12-20-b4884d9 /ray/python/build-wheel-manylinux2014.sh
         cp -rT /ray-mount /ray # copy new files back here
         find . | grep whl # testing
 

diff --git a/ci/env/install-minimal.sh b/ci/env/install-minimal.sh
@@ -14,6 +14,8 @@ else
         PYTHON_VERSION="3.9"
     elif [ "$1" = "3.10" ]; then
         PYTHON_VERSION="3.10"
+    elif [ "$1" = "3.11" ]; then
+        PYTHON_VERSION="3.11"
     else
         echo "Unsupported Python version."
         exit 1

diff --git a/doc/source/ray-overview/installation.rst b/doc/source/ray-overview/installation.rst
@@ -49,15 +49,16 @@ You can install the nightly Ray wheels via the following links. These daily rele
   # pip install -U LINK_TO_WHEEL.whl
 
 
-====================  ====================  =======================
-       Linux                 MacOS          Windows (beta)
-====================  ====================  =======================
-`Linux Python 3.10`_  `MacOS Python 3.10`_  `Windows Python 3.10`_
-`Linux Python 3.9`_   `MacOS Python 3.9`_   `Windows Python 3.9`_
-`Linux Python 3.8`_   `MacOS Python 3.8`_   `Windows Python 3.8`_
-`Linux Python 3.7`_   `MacOS Python 3.7`_   `Windows Python 3.7`_
-`Linux Python 3.6`_   `MacOS Python 3.6`_
-====================  ====================  =======================
+===================================  ====================  =======================
+       Linux                         MacOS                 Windows (beta)
+===================================  ====================  =======================
+`Linux Python 3.10`_                 `MacOS Python 3.10`_  `Windows Python 3.10`_
+`Linux Python 3.9`_                  `MacOS Python 3.9`_   `Windows Python 3.9`_
+`Linux Python 3.8`_                  `MacOS Python 3.8`_   `Windows Python 3.8`_
+`Linux Python 3.7`_                  `MacOS Python 3.7`_   `Windows Python 3.7`_
+`Linux Python 3.6`_                  `MacOS Python 3.6`_
+`Linux Python 3.11 (EXPERIMENTAL)`_
+===================================  ====================  =======================
 
 .. note::
 
@@ -68,6 +69,11 @@ You can install the nightly Ray wheels via the following links. These daily rele
 
   :ref:`Usage stats <ref-usage-stats>` collection is enabled by default (can be :ref:`disabled <usage-disable>`) for nightly wheels including both local clusters started via ``ray.init()`` and remote clusters via cli.
 
+.. note::
+
+  Python 3.11 support is experimental.
+
+.. _`Linux Python 3.11 (EXPERIMENTAL)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp311-cp311-manylinux2014_x86_64.whl
 .. _`Linux Python 3.10`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp310-cp310-manylinux2014_x86_64.whl
 .. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl
 .. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl

diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile
@@ -53,7 +53,7 @@ RUN sudo apt-get update -y && sudo apt-get upgrade -y \
     && $HOME/anaconda3/bin/conda clean -y --all \
     && $HOME/anaconda3/bin/pip install --no-cache-dir \
         flatbuffers \
-        cython==0.29.26 \
+        cython==0.29.32 \
         # Necessary for Dataset to work properly.
         numpy\>=1.20 \
         psutil \

diff --git a/docker/ray-worker-container/Dockerfile b/docker/ray-worker-container/Dockerfile
@@ -41,7 +41,7 @@ RUN apt-get update -y && sudo apt-get upgrade -y \
     && $HOME/anaconda3/bin/conda clean -y --all \
     && $HOME/anaconda3/bin/pip install --no-cache-dir \
         flatbuffers \
-        cython==0.29.26 \
+        cython==0.29.32 \
         numpy==1.15.4 \
         psutil \
     # To avoid the following error on Jenkins:

diff --git a/python/build-wheel-macos-arm64.sh b/python/build-wheel-macos-arm64.sh
@@ -71,7 +71,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
     # Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade.
     # TODO: Unpin after https://github.com/pypa/setuptools/issues/2849 is fixed.
     $PIP_CMD install --upgrade setuptools==58.4
-    $PIP_CMD install -q cython==0.29.26
+    $PIP_CMD install -q cython==0.29.32
     # Install wheel to avoid the error "invalid command 'bdist_wheel'".
     $PIP_CMD install -q wheel
     # Set the commit SHA in __init__.py.

diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh
@@ -18,23 +18,27 @@ PY_VERSIONS=("3.6.2"
              "3.7.0"
              "3.8.2"
              "3.9.1"
-             "3.10.4")
+             "3.10.4"
+             )
 PY_INSTS=("python-3.6.2-macosx10.6.pkg"
           "python-3.7.0-macosx10.6.pkg"
           "python-3.8.2-macosx10.9.pkg"
           "python-3.9.1-macosx10.9.pkg"
-          "python-3.10.4-macos11.pkg")
+          "python-3.10.4-macos11.pkg"
+          )
 PY_MMS=("3.6"
         "3.7"
         "3.8"
         "3.9"
-        "3.10")
+        "3.10"
+        )
 
 NUMPY_VERSIONS=("1.14.5"
                 "1.14.5"
                 "1.14.5"
                 "1.19.3"
-                "1.22.0")
+                "1.22.0"
+                )
 
 ./ci/env/install-bazel.sh
 
@@ -93,7 +97,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
     $PIP_CMD install -q setuptools_scm==3.1.0
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
-    $PIP_CMD install -q numpy=="$NUMPY_VERSION" cython==0.29.26
+    $PIP_CMD install -q numpy=="$NUMPY_VERSION" cython==0.29.32
     # Install wheel to avoid the error "invalid command 'bdist_wheel'".
     $PIP_CMD install -q wheel
     # Set the commit SHA in __init__.py.

diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh
@@ -16,12 +16,14 @@ PYTHONS=("cp36-cp36m"
          "cp37-cp37m"
          "cp38-cp38"
          "cp39-cp39"
-         "cp310-cp310")
+         "cp310-cp310"
+         "cp311-cp311")
 
 NUMPY_VERSIONS=("1.14.5"
                 "1.14.5"
                 "1.14.5"
                 "1.19.3"
+                "1.22.0"
                 "1.22.0")
 
 yum -y install unzip zip sudo
@@ -87,7 +89,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do
   pushd python
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
-    /opt/python/"${PYTHON}"/bin/pip install -q numpy=="${NUMPY_VERSION}" cython==0.29.26
+    /opt/python/"${PYTHON}"/bin/pip install -q numpy=="${NUMPY_VERSION}" cython==0.29.32
     # Set the commit SHA in __init__.py.
     if [ -n "$TRAVIS_COMMIT" ]; then
       sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak

diff --git a/python/build-wheel-windows.sh b/python/build-wheel-windows.sh
@@ -8,7 +8,8 @@ WORKSPACE_DIR="${ROOT_DIR}/.."
 PY_VERSIONS=("3.7"
              "3.8"
              "3.9"
-             "3.10")
+             "3.10"
+             )
 
 bazel_preclean() {
   "${WORKSPACE_DIR}"/ci/run/bazel.py preclean "mnemonic(\"Genrule\", deps(//:*))"

diff --git a/python/ray/_raylet.pxd b/python/ray/_raylet.pxd
@@ -67,7 +67,8 @@ cdef extern from "Python.h":
 
     # You can find the cpython definition in Include/cpython/pystate.h#L59
     ctypedef struct CPyThreadState "PyThreadState":
-        int recursion_depth
+        int recursion_limit
+        int recursion_remaining
 
     # From Include/ceveal.h#67
     int Py_GetRecursionLimit()

diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx
@@ -261,9 +261,19 @@ cdef increase_recursion_limit():
     """Double the recusion limit if current depth is close to the limit"""
     cdef:
         CPyThreadState * s = <CPyThreadState *> PyThreadState_Get()
-        int current_depth = s.recursion_depth
         int current_limit = Py_GetRecursionLimit()
         int new_limit = current_limit * 2
+        cdef extern from *:
+            """
+#if PY_VERSION_HEX >= 0x30B00A4
+    #define CURRENT_DEPTH(x)  ((x)->recursion_limit - (x)->recursion_remaining)
+#else
+    #define CURRENT_DEPTH(x)  ((x)->recursion_depth)
+#endif
+            """
+            int CURRENT_DEPTH(CPyThreadState *x)
+
+        int current_depth = CURRENT_DEPTH(s)
 
     if current_limit - current_depth < 500:
         Py_SetRecursionLimit(new_limit)

diff --git a/python/ray/autoscaler/aws/development-example.yaml b/python/ray/autoscaler/aws/development-example.yaml
@@ -59,7 +59,7 @@ setup_commands:
     - git clone https://github.com/ray-project/ray || true
     - ray/ci/env/install-bazel.sh
     - cd ray/python/ray/dashboard/client; npm ci; npm run build
-    - pip install boto3==1.4.8 cython==0.29.26 aiohttp grpcio psutil setproctitle
+    - pip install boto3==1.4.8 cython==0.29.32 aiohttp grpcio psutil setproctitle
     - cd ray/python; pip install -e . --verbose
 
 # Command to start ray on the head node. You don't need to change this.