diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml index a224f1796cba..359ab0286237 100644 --- a/.buildkite/pipeline.build.yml +++ b/.buildkite/pipeline.build.yml @@ -337,6 +337,14 @@ - ./ci/ci.sh test_minimal 3.10 - ./ci/ci.sh test_latest_core_dependencies 3.10 +- label: ":python: Minimal install 3.11" + conditions: ["RAY_CI_PYTHON_AFFECTED"] + instance_size: medium + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - ./ci/ci.sh test_minimal 3.11 + - ./ci/ci.sh test_latest_core_dependencies 3.11 + - label: ":python: Default install" conditions: ["RAY_CI_PYTHON_AFFECTED"] instance_size: small diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl index 3f8462b77272..272c469c793c 100644 --- a/bazel/ray_deps_setup.bzl +++ b/bazel/ray_deps_setup.bzl @@ -194,8 +194,8 @@ def ray_deps_setup(): auto_http_archive( name = "cython", build_file = True, - url = "https://github.com/cython/cython/archive/3028e8c7ac296bc848d996e397c3354b3dbbd431.tar.gz", - sha256 = "31ea23c2231ddee8572a2a5effd54952e16a1b44e9a4cb3eb645418f8accf20d", + url = "https://github.com/cython/cython/archive/c48361d0a0969206e227ec016f654c9d941c2b69.tar.gz", + sha256 = "37c466fea398da9785bc37fe16f1455d2645d21a72e402103991d9e2fa1c6ff3", ) auto_http_archive( diff --git a/ci/build/build-docker-images.py b/ci/build/build-docker-images.py index d962a1a2c5ae..fd8745af00e1 100644 --- a/ci/build/build-docker-images.py +++ b/ci/build/build-docker-images.py @@ -620,7 +620,7 @@ def push_readmes(merge_build: bool): default="py37", nargs="*", help="Which python versions to build. " - "Must be in (py36, py37, py38, py39, py310)", + "Must be in (py36, py37, py38, py39, py310, py311)", ) parser.add_argument( "--device-types", diff --git a/ci/build/test-wheels.sh b/ci/build/test-wheels.sh index dcf7219ec80f..5f51f8d69f9e 100755 --- a/ci/build/test-wheels.sh +++ b/ci/build/test-wheels.sh @@ -100,7 +100,8 @@ elif [[ "$platform" == "macosx" ]]; then "3.7" "3.8" "3.9" - "3.10") + "3.10" + ) for ((i=0; i<${#PY_MMS[@]}; ++i)); do PY_MM="${PY_MMS[i]}" diff --git a/ci/ci.sh b/ci/ci.sh index af7350e04cfd..b69beb109f81 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -414,8 +414,7 @@ validate_wheels_commit_str() { continue fi - folder=${basename%%-cp*} - WHL_COMMIT=$(unzip -p "$whl" "${folder}.data/purelib/ray/__init__.py" | grep "__commit__" | awk -F'"' '{print $2}') + WHL_COMMIT=$(unzip -p "$whl" | grep "^__commit__" | awk -F'"' '{print $2}') if [ "${WHL_COMMIT}" != "${EXPECTED_COMMIT}" ]; then echo "Error: Observed wheel commit (${WHL_COMMIT}) is not expected commit (${EXPECTED_COMMIT}). Aborting." @@ -459,7 +458,7 @@ build_wheels() { # This command should be kept in sync with ray/python/README-building-wheels.md, # except the "${MOUNT_BAZEL_CACHE[@]}" part. docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \ - quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh + quay.io/pypa/manylinux2014_x86_64:2022-12-20-b4884d9 /ray/python/build-wheel-manylinux2014.sh else rm -rf /ray-mount/* rm -rf /ray-mount/.whl || true @@ -469,7 +468,7 @@ build_wheels() { docker run --rm -v /ray:/ray-mounted ubuntu:focal ls / docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /ray-mounted docker run --rm -w /ray -v /ray:/ray "${MOUNT_BAZEL_CACHE[@]}" \ - quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh + quay.io/pypa/manylinux2014_x86_64:2022-12-20-b4884d9 /ray/python/build-wheel-manylinux2014.sh cp -rT /ray-mount /ray # copy new files back here find . | grep whl # testing diff --git a/ci/env/install-minimal.sh b/ci/env/install-minimal.sh index 9c3293ca9d73..8d451739ea3d 100755 --- a/ci/env/install-minimal.sh +++ b/ci/env/install-minimal.sh @@ -14,6 +14,8 @@ else PYTHON_VERSION="3.9" elif [ "$1" = "3.10" ]; then PYTHON_VERSION="3.10" + elif [ "$1" = "3.11" ]; then + PYTHON_VERSION="3.11" else echo "Unsupported Python version." exit 1 diff --git a/doc/source/ray-overview/installation.rst b/doc/source/ray-overview/installation.rst index 5bd939fe476f..975ce6126f1e 100644 --- a/doc/source/ray-overview/installation.rst +++ b/doc/source/ray-overview/installation.rst @@ -49,15 +49,16 @@ You can install the nightly Ray wheels via the following links. These daily rele # pip install -U LINK_TO_WHEEL.whl -==================== ==================== ======================= - Linux MacOS Windows (beta) -==================== ==================== ======================= -`Linux Python 3.10`_ `MacOS Python 3.10`_ `Windows Python 3.10`_ -`Linux Python 3.9`_ `MacOS Python 3.9`_ `Windows Python 3.9`_ -`Linux Python 3.8`_ `MacOS Python 3.8`_ `Windows Python 3.8`_ -`Linux Python 3.7`_ `MacOS Python 3.7`_ `Windows Python 3.7`_ -`Linux Python 3.6`_ `MacOS Python 3.6`_ -==================== ==================== ======================= +=================================== ==================== ======================= + Linux MacOS Windows (beta) +=================================== ==================== ======================= +`Linux Python 3.10`_ `MacOS Python 3.10`_ `Windows Python 3.10`_ +`Linux Python 3.9`_ `MacOS Python 3.9`_ `Windows Python 3.9`_ +`Linux Python 3.8`_ `MacOS Python 3.8`_ `Windows Python 3.8`_ +`Linux Python 3.7`_ `MacOS Python 3.7`_ `Windows Python 3.7`_ +`Linux Python 3.6`_ `MacOS Python 3.6`_ +`Linux Python 3.11 (EXPERIMENTAL)`_ +=================================== ==================== ======================= .. note:: @@ -68,6 +69,11 @@ You can install the nightly Ray wheels via the following links. These daily rele :ref:`Usage stats ` collection is enabled by default (can be :ref:`disabled `) for nightly wheels including both local clusters started via ``ray.init()`` and remote clusters via cli. +.. note:: + + Python 3.11 support is experimental. + +.. _`Linux Python 3.11 (EXPERIMENTAL)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp311-cp311-manylinux2014_x86_64.whl .. _`Linux Python 3.10`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp310-cp310-manylinux2014_x86_64.whl .. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl .. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 440bf3208eb4..77c8d79e969c 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -53,7 +53,7 @@ RUN sudo apt-get update -y && sudo apt-get upgrade -y \ && $HOME/anaconda3/bin/conda clean -y --all \ && $HOME/anaconda3/bin/pip install --no-cache-dir \ flatbuffers \ - cython==0.29.26 \ + cython==0.29.32 \ # Necessary for Dataset to work properly. numpy\>=1.20 \ psutil \ diff --git a/docker/ray-worker-container/Dockerfile b/docker/ray-worker-container/Dockerfile index f7679219e381..e340b6af0b62 100644 --- a/docker/ray-worker-container/Dockerfile +++ b/docker/ray-worker-container/Dockerfile @@ -41,7 +41,7 @@ RUN apt-get update -y && sudo apt-get upgrade -y \ && $HOME/anaconda3/bin/conda clean -y --all \ && $HOME/anaconda3/bin/pip install --no-cache-dir \ flatbuffers \ - cython==0.29.26 \ + cython==0.29.32 \ numpy==1.15.4 \ psutil \ # To avoid the following error on Jenkins: diff --git a/python/build-wheel-macos-arm64.sh b/python/build-wheel-macos-arm64.sh index d9726ed2e77b..273dc57c56fa 100755 --- a/python/build-wheel-macos-arm64.sh +++ b/python/build-wheel-macos-arm64.sh @@ -71,7 +71,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do # Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade. # TODO: Unpin after https://github.com/pypa/setuptools/issues/2849 is fixed. $PIP_CMD install --upgrade setuptools==58.4 - $PIP_CMD install -q cython==0.29.26 + $PIP_CMD install -q cython==0.29.32 # Install wheel to avoid the error "invalid command 'bdist_wheel'". $PIP_CMD install -q wheel # Set the commit SHA in __init__.py. diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 0acf469be19c..1cef6d9e4455 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -18,23 +18,27 @@ PY_VERSIONS=("3.6.2" "3.7.0" "3.8.2" "3.9.1" - "3.10.4") + "3.10.4" + ) PY_INSTS=("python-3.6.2-macosx10.6.pkg" "python-3.7.0-macosx10.6.pkg" "python-3.8.2-macosx10.9.pkg" "python-3.9.1-macosx10.9.pkg" - "python-3.10.4-macos11.pkg") + "python-3.10.4-macos11.pkg" + ) PY_MMS=("3.6" "3.7" "3.8" "3.9" - "3.10") + "3.10" + ) NUMPY_VERSIONS=("1.14.5" "1.14.5" "1.14.5" "1.19.3" - "1.22.0") + "1.22.0" + ) ./ci/env/install-bazel.sh @@ -93,7 +97,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $PIP_CMD install -q setuptools_scm==3.1.0 # Fix the numpy version because this will be the oldest numpy version we can # support. - $PIP_CMD install -q numpy=="$NUMPY_VERSION" cython==0.29.26 + $PIP_CMD install -q numpy=="$NUMPY_VERSION" cython==0.29.32 # Install wheel to avoid the error "invalid command 'bdist_wheel'". $PIP_CMD install -q wheel # Set the commit SHA in __init__.py. diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh index 92c7b8fa8203..aa975cedfe94 100755 --- a/python/build-wheel-manylinux2014.sh +++ b/python/build-wheel-manylinux2014.sh @@ -16,12 +16,14 @@ PYTHONS=("cp36-cp36m" "cp37-cp37m" "cp38-cp38" "cp39-cp39" - "cp310-cp310") + "cp310-cp310" + "cp311-cp311") NUMPY_VERSIONS=("1.14.5" "1.14.5" "1.14.5" "1.19.3" + "1.22.0" "1.22.0") yum -y install unzip zip sudo @@ -87,7 +89,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do pushd python # Fix the numpy version because this will be the oldest numpy version we can # support. - /opt/python/"${PYTHON}"/bin/pip install -q numpy=="${NUMPY_VERSION}" cython==0.29.26 + /opt/python/"${PYTHON}"/bin/pip install -q numpy=="${NUMPY_VERSION}" cython==0.29.32 # Set the commit SHA in __init__.py. if [ -n "$TRAVIS_COMMIT" ]; then sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak diff --git a/python/build-wheel-windows.sh b/python/build-wheel-windows.sh index 530bcdd2ccf3..b88e74a441ba 100755 --- a/python/build-wheel-windows.sh +++ b/python/build-wheel-windows.sh @@ -8,7 +8,8 @@ WORKSPACE_DIR="${ROOT_DIR}/.." PY_VERSIONS=("3.7" "3.8" "3.9" - "3.10") + "3.10" + ) bazel_preclean() { "${WORKSPACE_DIR}"/ci/run/bazel.py preclean "mnemonic(\"Genrule\", deps(//:*))" diff --git a/python/ray/_raylet.pxd b/python/ray/_raylet.pxd index f6f34a24b55f..6af1879a5d8a 100644 --- a/python/ray/_raylet.pxd +++ b/python/ray/_raylet.pxd @@ -67,7 +67,8 @@ cdef extern from "Python.h": # You can find the cpython definition in Include/cpython/pystate.h#L59 ctypedef struct CPyThreadState "PyThreadState": - int recursion_depth + int recursion_limit + int recursion_remaining # From Include/ceveal.h#67 int Py_GetRecursionLimit() diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index 7ed8dffd4ae2..892bb4aa2d0e 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -261,9 +261,19 @@ cdef increase_recursion_limit(): """Double the recusion limit if current depth is close to the limit""" cdef: CPyThreadState * s = PyThreadState_Get() - int current_depth = s.recursion_depth int current_limit = Py_GetRecursionLimit() int new_limit = current_limit * 2 + cdef extern from *: + """ +#if PY_VERSION_HEX >= 0x30B00A4 + #define CURRENT_DEPTH(x) ((x)->recursion_limit - (x)->recursion_remaining) +#else + #define CURRENT_DEPTH(x) ((x)->recursion_depth) +#endif + """ + int CURRENT_DEPTH(CPyThreadState *x) + + int current_depth = CURRENT_DEPTH(s) if current_limit - current_depth < 500: Py_SetRecursionLimit(new_limit) diff --git a/python/ray/autoscaler/aws/development-example.yaml b/python/ray/autoscaler/aws/development-example.yaml index 30ac1b4ffb9e..467e755247ee 100644 --- a/python/ray/autoscaler/aws/development-example.yaml +++ b/python/ray/autoscaler/aws/development-example.yaml @@ -59,7 +59,7 @@ setup_commands: - git clone https://github.com/ray-project/ray || true - ray/ci/env/install-bazel.sh - cd ray/python/ray/dashboard/client; npm ci; npm run build - - pip install boto3==1.4.8 cython==0.29.26 aiohttp grpcio psutil setproctitle + - pip install boto3==1.4.8 cython==0.29.32 aiohttp grpcio psutil setproctitle - cd ray/python; pip install -e . --verbose # Command to start ray on the head node. You don't need to change this. diff --git a/python/ray/cloudpickle/cloudpickle.py b/python/ray/cloudpickle/cloudpickle.py index 43003b3a5b16..9c67ec64c861 100644 --- a/python/ray/cloudpickle/cloudpickle.py +++ b/python/ray/cloudpickle/cloudpickle.py @@ -41,7 +41,6 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ -from __future__ import print_function import builtins import dis @@ -57,7 +56,7 @@ from .compat import pickle from collections import OrderedDict -from typing import Generic, Union, Tuple, Callable +from typing import ClassVar, Generic, Union, Tuple, Callable from pickle import _getattribute try: # pragma: no branch @@ -66,11 +65,6 @@ except ImportError: _typing_extensions = Literal = Final = None -if sys.version_info >= (3, 5, 3): - from typing import ClassVar -else: # pragma: no cover - ClassVar = None - if sys.version_info >= (3, 8): from types import CellType else: @@ -328,11 +322,10 @@ def _extract_code_globals(co): """ out_names = _extract_code_globals_cache.get(co) if out_names is None: - names = co.co_names # We use a dict with None values instead of a set to get a # deterministic order (assuming Python 3.6+) and avoid introducing # non-deterministic pickle bytes as a results. - out_names = {names[oparg]: None for _, oparg in _walk_global_ops(co)} + out_names = {name: None for name in _walk_global_ops(co)} # Declaring a function inside another one using the "def ..." # syntax generates a constant code object corresponding to the one @@ -412,8 +405,7 @@ def cell_set(cell, value): In Python3.7, cell_contents is writeable, so setting the contents of a cell can be done simply using - >>> cell = ... # doctest: +SKIP - >>> cell.cell_contents = value # doctest: +SKIP + >>> cell.cell_contents = value In earlier Python3 versions, the cell_contents attribute of a cell is read only, but this limitation can be worked around by leveraging the Python 3 @@ -527,13 +519,12 @@ def _builtin_type(name): def _walk_global_ops(code): """ - Yield (opcode, argument number) tuples for all - global-referencing instructions in *code*. + Yield referenced name for all global-referencing instructions in *code*. """ for instr in dis.get_instructions(code): op = instr.opcode if op in GLOBAL_OPS: - yield op, instr.arg + yield instr.argval def _extract_class_dict(cls): @@ -621,44 +612,20 @@ def parametrized_type_hint_getinitargs(obj): elif type(obj) is type(ClassVar): initargs = (ClassVar, obj.__type__) elif type(obj) is type(Generic): - parameters = obj.__parameters__ - if len(obj.__parameters__) > 0: - # in early Python 3.5, __parameters__ was sometimes - # preferred to __args__ - initargs = (obj.__origin__, parameters) - - else: - initargs = (obj.__origin__, obj.__args__) + initargs = (obj.__origin__, obj.__args__) elif type(obj) is type(Union): - if sys.version_info < (3, 5, 3): # pragma: no cover - initargs = (Union, obj.__union_params__) - else: - initargs = (Union, obj.__args__) + initargs = (Union, obj.__args__) elif type(obj) is type(Tuple): - if sys.version_info < (3, 5, 3): # pragma: no cover - initargs = (Tuple, obj.__tuple_params__) - else: - initargs = (Tuple, obj.__args__) + initargs = (Tuple, obj.__args__) elif type(obj) is type(Callable): - if sys.version_info < (3, 5, 3): # pragma: no cover - args = obj.__args__ - result = obj.__result__ - if args != Ellipsis: - if isinstance(args, tuple): - args = list(args) - else: - args = [args] + (*args, result) = obj.__args__ + if len(args) == 1 and args[0] is Ellipsis: + args = Ellipsis else: - (*args, result) = obj.__args__ - if len(args) == 1 and args[0] is Ellipsis: - args = Ellipsis - else: - args = list(args) + args = list(args) initargs = (Callable, (args, result)) else: # pragma: no cover - raise pickle.PicklingError( - "Cloudpickle Error: Unknown type {}".format(type(obj)) - ) + raise pickle.PicklingError(f"Cloudpickle Error: Unknown type {type(obj)}") return initargs @@ -739,7 +706,7 @@ def instance(cls): @instance -class _empty_cell_value(object): +class _empty_cell_value: """sentinel for empty closures""" @classmethod @@ -768,7 +735,7 @@ def _fill_function(*args): keys = ["globals", "defaults", "dict", "module", "closure_values"] state = dict(zip(keys, args[1:])) else: - raise ValueError("Unexpected _fill_value arguments: %r" % (args,)) + raise ValueError(f"Unexpected _fill_value arguments: {args!r}") # - At pickling time, any dynamic global variable used by func is # serialized by value (in state['globals']). @@ -812,6 +779,12 @@ def _fill_function(*args): return func +def _make_function(code, globals, name, argdefs, closure): + # Setting __builtins__ in globals is needed for nogil CPython. + globals["__builtins__"] = __builtins__ + return types.FunctionType(code, globals, name, argdefs, closure) + + def _make_empty_cell(): if False: # trick the compiler into creating an empty cell in our lambda @@ -940,18 +913,13 @@ def _make_typevar(name, bound, constraints, covariant, contravariant, class_trac def _decompose_typevar(obj): - try: - class_tracker_id = _get_or_create_tracker_id(obj) - except TypeError: # pragma: nocover - # TypeVar instances are not weakref-able in Python 3.5.3 - class_tracker_id = None return ( obj.__name__, obj.__bound__, obj.__constraints__, obj.__covariant__, obj.__contravariant__, - class_tracker_id, + _get_or_create_tracker_id(obj), ) @@ -969,13 +937,13 @@ def _typevar_reduce(obj): def _get_bases(typ): - if '__orig_bases__' in getattr(typ, '__dict__', {}): + if "__orig_bases__" in getattr(typ, "__dict__", {}): # For generic types (see PEP 560) # Note that simply checking `hasattr(typ, '__orig_bases__')` is not # correct. Subclasses of a fully-parameterized generic class does not # have `__orig_bases__` defined, but `hasattr(typ, '__orig_bases__')` # will return True because it's defined in the base class. - bases_attr = '__orig_bases__' + bases_attr = "__orig_bases__" else: # For regular class objects bases_attr = "__bases__" diff --git a/python/ray/cloudpickle/cloudpickle_fast.py b/python/ray/cloudpickle/cloudpickle_fast.py index 18c000e91528..b3c20b9e4ed4 100644 --- a/python/ray/cloudpickle/cloudpickle_fast.py +++ b/python/ray/cloudpickle/cloudpickle_fast.py @@ -27,19 +27,38 @@ from .compat import pickle, Pickler from .cloudpickle import ( - _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, - _find_imported_submodules, _get_cell_contents, _should_pickle_by_reference, - _builtin_type, _get_or_create_tracker_id, _make_skeleton_class, - _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, - _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, - _is_parametrized_type_hint, PYPY, cell_set, - parametrized_type_hint_getinitargs, _create_parametrized_type_hint, + _extract_code_globals, + _BUILTIN_TYPE_NAMES, + DEFAULT_PROTOCOL, + _find_imported_submodules, + _get_cell_contents, + _should_pickle_by_reference, + _builtin_type, + _get_or_create_tracker_id, + _make_skeleton_class, + _make_skeleton_enum, + _extract_class_dict, + dynamic_subimport, + subimport, + _typevar_reduce, + _get_bases, + _make_cell, + _make_empty_cell, + CellType, + _is_parametrized_type_hint, + PYPY, + cell_set, + parametrized_type_hint_getinitargs, + _create_parametrized_type_hint, builtin_code_type, - _make_dict_keys, _make_dict_values, _make_dict_items, + _make_dict_keys, + _make_dict_values, + _make_dict_items, + _make_function, ) -if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: +if pickle.HIGHEST_PROTOCOL >= 5: # Shorthands similar to pickle.dump/pickle.dumps def dump(obj, file, protocol=None, buffer_callback=None): @@ -52,9 +71,7 @@ def dump(obj, file, protocol=None, buffer_callback=None): Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ - CloudPickler( - file, protocol=protocol, buffer_callback=buffer_callback - ).dump(obj) + CloudPickler(file, protocol=protocol, buffer_callback=buffer_callback).dump(obj) def dumps(obj, protocol=None, buffer_callback=None): """Serialize obj as a string of bytes allocated in memory @@ -67,9 +84,7 @@ def dumps(obj, protocol=None, buffer_callback=None): compatibility with older versions of Python. """ with io.BytesIO() as file: - cp = CloudPickler( - file, protocol=protocol, buffer_callback=buffer_callback - ) + cp = CloudPickler(file, protocol=protocol, buffer_callback=buffer_callback) cp.dump(obj) return file.getvalue() @@ -109,23 +124,37 @@ def dumps(obj, protocol=None): # COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS # ------------------------------------------------- + def _class_getnewargs(obj): type_kwargs = {} if "__slots__" in obj.__dict__: type_kwargs["__slots__"] = obj.__slots__ - __dict__ = obj.__dict__.get('__dict__', None) + __dict__ = obj.__dict__.get("__dict__", None) if isinstance(__dict__, property): - type_kwargs['__dict__'] = __dict__ - - return (type(obj), obj.__name__, _get_bases(obj), type_kwargs, - _get_or_create_tracker_id(obj), None) + type_kwargs["__dict__"] = __dict__ + + return ( + type(obj), + obj.__name__, + _get_bases(obj), + type_kwargs, + _get_or_create_tracker_id(obj), + None, + ) def _enum_getnewargs(obj): - members = dict((e.name, e.value) for e in obj) - return (obj.__bases__, obj.__name__, obj.__qualname__, members, - obj.__module__, _get_or_create_tracker_id(obj), None) + members = {e.name: e.value for e in obj} + return ( + obj.__bases__, + obj.__name__, + obj.__qualname__, + members, + obj.__module__, + _get_or_create_tracker_id(obj), + None, + ) # COLLECTION OF OBJECTS RECONSTRUCTORS @@ -155,12 +184,12 @@ def _function_getstate(func): } f_globals_ref = _extract_code_globals(func.__code__) - f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in - func.__globals__} + f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in func.__globals__} closure_values = ( list(map(_get_cell_contents, func.__closure__)) - if func.__closure__ is not None else () + if func.__closure__ is not None + else () ) # Extract currently-imported submodules used by func. Storing these modules @@ -168,7 +197,8 @@ def _function_getstate(func): # trigger the side effect of importing these modules at unpickling time # (which is necessary for func to work correctly once depickled) slotstate["_cloudpickle_submodules"] = _find_imported_submodules( - func.__code__, itertools.chain(f_globals.values(), closure_values)) + func.__code__, itertools.chain(f_globals.values(), closure_values) + ) slotstate["__globals__"] = f_globals state = func.__dict__ @@ -177,29 +207,26 @@ def _function_getstate(func): def _class_getstate(obj): clsdict = _extract_class_dict(obj) - clsdict.pop('__weakref__', None) + clsdict.pop("__weakref__", None) if issubclass(type(obj), abc.ABCMeta): # If obj is an instance of an ABCMeta subclass, don't pickle the # cache/negative caches populated during isinstance/issubclass # checks, but pickle the list of registered subclasses of obj. - clsdict.pop('_abc_cache', None) - clsdict.pop('_abc_negative_cache', None) - clsdict.pop('_abc_negative_cache_version', None) - # these are generated by some thirdparty libraries - clsdict.pop('_abc_generic_negative_cache', None) - clsdict.pop('_abc_generic_negative_cache_version', None) - - registry = clsdict.pop('_abc_registry', None) + clsdict.pop("_abc_cache", None) + clsdict.pop("_abc_negative_cache", None) + clsdict.pop("_abc_negative_cache_version", None) + registry = clsdict.pop("_abc_registry", None) if registry is None: - if hasattr(abc, '_get_dump'): + if hasattr(abc, "_get_dump"): # in Python3.7+, the abc caches and registered subclasses of a # class are bundled into the single _abc_impl attribute - clsdict.pop('_abc_impl', None) + clsdict.pop("_abc_impl", None) (registry, _, _, _) = abc._get_dump(obj) - clsdict["_abc_impl"] = [subclass_weakref() - for subclass_weakref in registry] + clsdict["_abc_impl"] = [ + subclass_weakref() for subclass_weakref in registry + ] else: # FIXME(suquark): The upstream cloudpickle cannot work in Ray # because sometimes both '_abc_registry' and '_get_dump' does @@ -221,7 +248,7 @@ def _class_getstate(obj): for k in obj.__slots__: clsdict.pop(k, None) - clsdict.pop('__dict__', None) # unpicklable property object + clsdict.pop("__dict__", None) # unpicklable property object return (clsdict, {}) @@ -229,12 +256,16 @@ def _class_getstate(obj): def _enum_getstate(obj): clsdict, slotstate = _class_getstate(obj) - members = dict((e.name, e.value) for e in obj) + members = {e.name: e.value for e in obj} # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: # Those attributes are already handled by the metaclass. - for attrname in ["_generate_next_value_", "_member_names_", - "_member_map_", "_member_type_", - "_value2member_map_"]: + for attrname in [ + "_generate_next_value_", + "_member_names_", + "_member_map_", + "_member_type_", + "_value2member_map_", + ]: clsdict.pop(attrname, None) for member in members: clsdict.pop(member) @@ -253,37 +284,120 @@ def _enum_getstate(obj): # obj.__reduce__), some do not. The following methods were created to "fill # these holes". + def _code_reduce(obj): """codeobject reducer""" - if hasattr(obj, "co_linetable"): # pragma: no branch + # If you are not sure about the order of arguments, take a look at help + # of the specific type from types, for example: + # >>> from types import CodeType + # >>> help(CodeType) + if hasattr(obj, "co_exceptiontable"): # pragma: no branch + # Python 3.11 and later: there are some new attributes + # related to the enhanced exceptions. + args = ( + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + obj.co_filename, + obj.co_name, + obj.co_qualname, + obj.co_firstlineno, + obj.co_linetable, + obj.co_exceptiontable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_linetable"): # pragma: no branch # Python 3.10 and later: obj.co_lnotab is deprecated and constructor # expects obj.co_linetable instead. args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_linetable, obj.co_freevars, - obj.co_cellvars + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + obj.co_filename, + obj.co_name, + obj.co_firstlineno, + obj.co_linetable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_nmeta"): # pragma: no cover + # "nogil" Python: modified attributes from 3.9 + args = ( + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_framesize, + obj.co_ndefaultargs, + obj.co_nmeta, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_varnames, + obj.co_filename, + obj.co_name, + obj.co_firstlineno, + obj.co_lnotab, + obj.co_exc_handlers, + obj.co_jump_table, + obj.co_freevars, + obj.co_cellvars, + obj.co_free2reg, + obj.co_cell2reg, ) elif hasattr(obj, "co_posonlyargcount"): # Backward compat for 3.9 and older args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + obj.co_filename, + obj.co_name, + obj.co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, ) else: # Backward compat for even older versions of Python args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, - obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, - obj.co_names, obj.co_varnames, obj.co_filename, - obj.co_name, obj.co_firstlineno, obj.co_lnotab, - obj.co_freevars, obj.co_cellvars + obj.co_argcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + obj.co_filename, + obj.co_name, + obj.co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, ) return types.CodeType, args @@ -295,7 +409,7 @@ def _cell_reduce(obj): except ValueError: # cell is empty return _make_empty_cell, () else: - return _make_cell, (obj.cell_contents, ) + return _make_cell, (obj.cell_contents,) def _classmethod_reduce(obj): @@ -320,13 +434,10 @@ def _file_reduce(obj): if obj.closed: raise pickle.PicklingError("Cannot pickle closed files") if hasattr(obj, "isatty") and obj.isatty(): - raise pickle.PicklingError( - "Cannot pickle files that map to tty objects" - ) + raise pickle.PicklingError("Cannot pickle files that map to tty objects") if "r" not in obj.mode and "+" not in obj.mode: raise pickle.PicklingError( - "Cannot pickle files that are not opened for reading: %s" - % obj.mode + "Cannot pickle files that are not opened for reading: %s" % obj.mode ) name = obj.name @@ -371,7 +482,7 @@ def _module_reduce(obj): # reason, we do not attempt to pickle the "__builtins__" entry, and # restore a default value for it at unpickling time. state = obj.__dict__.copy() - state.pop('__builtins__', None) + state.pop("__builtins__", None) return dynamic_subimport, (obj.__name__, state) @@ -405,13 +516,21 @@ def _dynamic_class_reduce(obj): """ if Enum is not None and issubclass(obj, Enum): return ( - _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj), - None, None, _class_setstate + _make_skeleton_enum, + _enum_getnewargs(obj), + _enum_getstate(obj), + None, + None, + _class_setstate, ) else: return ( - _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj), - None, None, _class_setstate + _make_skeleton_class, + _class_getnewargs(obj), + _class_getstate(obj), + None, + None, + _class_setstate, ) @@ -434,18 +553,18 @@ def _dict_keys_reduce(obj): # Safer not to ship the full dict as sending the rest might # be unintended and could potentially cause leaking of # sensitive information - return _make_dict_keys, (list(obj), ) + return _make_dict_keys, (list(obj),) def _dict_values_reduce(obj): # Safer not to ship the full dict as sending the rest might # be unintended and could potentially cause leaking of # sensitive information - return _make_dict_values, (list(obj), ) + return _make_dict_values, (list(obj),) def _dict_items_reduce(obj): - return _make_dict_items, (dict(obj), ) + return _make_dict_items, (dict(obj),) def _odict_keys_reduce(obj): @@ -521,13 +640,6 @@ def _class_setstate(obj, state): return obj -def _ufunc_reduce(func): - # This function comes from https://github.com/numpy/numpy/pull/17289. - # It fixes the original improper numpy ufunc serializer. - # numpy >= 1.20.0 uses this function by default. - return func.__name__ - - class CloudPickler(Pickler): # set of reducers defined and used by cloudpickle (private) _dispatch_table = {} @@ -552,16 +664,12 @@ class CloudPickler(Pickler): _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce - + _dispatch_table[abc.abstractmethod] = _classmethod_reduce + _dispatch_table[abc.abstractclassmethod] = _classmethod_reduce + _dispatch_table[abc.abstractstaticmethod] = _classmethod_reduce + _dispatch_table[abc.abstractproperty] = _property_reduce dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) - # TODO(suquark): Remove this patch when we use numpy >= 1.20.0 by default. - # We import 'numpy.core' here, so numpy would register the - # ufunc serializer to 'copyreg.dispatch_table' before we override it. - import numpy.core - import numpy - # Override the original numpy ufunc serializer. - dispatch_table[numpy.ufunc] = _ufunc_reduce # function reducers are defined as instance methods of CloudPickler # objects, as they rely on a CloudPickler attribute (globals_ref) @@ -569,8 +677,7 @@ def _dynamic_function_reduce(self, func): """Reduce a function that is not pickleable via attribute lookup.""" newargs = self._function_getnewargs(func) state = _function_getstate(func) - return (types.FunctionType, newargs, state, None, None, - _function_setstate) + return (_make_function, newargs, state, None, None, _function_setstate) def _function_reduce(self, obj): """Reducer for function objects. @@ -617,8 +724,7 @@ def _function_getnewargs(self, func): if func.__closure__ is None: closure = None else: - closure = tuple( - _make_empty_cell() for _ in range(len(code.co_freevars))) + closure = tuple(_make_empty_cell() for _ in range(len(code.co_freevars))) return code, base_globals, None, None, closure @@ -628,14 +734,42 @@ def dump(self, obj): except RuntimeError as e: if "recursion" in e.args[0]: msg = ( - "Could not pickle object as excessively deep recursion " - "required." + "Could not pickle object as excessively deep recursion " "required." ) raise pickle.PicklingError(msg) from e else: raise if pickle.HIGHEST_PROTOCOL >= 5: + + def __init__(self, file, protocol=None, buffer_callback=None): + if protocol is None: + protocol = DEFAULT_PROTOCOL + Pickler.__init__( + self, file, protocol=protocol, buffer_callback=buffer_callback + ) + # map functions __globals__ attribute ids, to ensure that functions + # sharing the same global namespace at pickling time also share + # their global namespace at unpickling time. + self.globals_ref = {} + self.proto = int(protocol) + + else: + + def __init__(self, file, protocol=None): + if protocol is None: + protocol = DEFAULT_PROTOCOL + Pickler.__init__(self, file, protocol=protocol) + # map functions __globals__ attribute ids, to ensure that functions + # sharing the same global namespace at pickling time also share + # their global namespace at unpickling time. + self.globals_ref = {} + assert hasattr(self, "proto") + + if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: + # Pickler is the C implementation of the CPython pickler and therefore + # we rely on reduce_override method to customize the pickler behavior. + # `CloudPickler.dispatch` is only left for backward compatibility - note # that when using protocol 5, `CloudPickler.dispatch` is not an # extension of `Pickler.dispatch` dictionary, because CloudPickler @@ -656,17 +790,6 @@ def dump(self, obj): # availability of both notions coincide on CPython's pickle and the # pickle5 backport, but it may not be the case anymore when pypy # implements protocol 5 - def __init__(self, file, protocol=None, buffer_callback=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - Pickler.__init__( - self, file, protocol=protocol, buffer_callback=buffer_callback - ) - # map functions __globals__ attribute ids, to ensure that functions - # sharing the same global namespace at pickling time also share - # their global namespace at unpickling time. - self.globals_ref = {} - self.proto = int(protocol) def reducer_override(self, obj): """Type-agnostic reducing callback for function and classes. @@ -700,16 +823,13 @@ def reducer_override(self, obj): reducers, such as Exceptions. See https://github.com/cloudpipe/cloudpickle/issues/248 """ - if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch - try: - return ( - _create_parametrized_type_hint, - parametrized_type_hint_getinitargs(obj) - ) - except pickle.PicklingError: - # There are some false positive cases in '_is_parametrized_type_hint'. - # We should not fail early for these false positive cases. - pass + if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint( + obj + ): # noqa # pragma: no branch + return ( + _create_parametrized_type_hint, + parametrized_type_hint_getinitargs(obj), + ) t = type(obj) try: is_anyclass = issubclass(t, type) @@ -732,23 +852,25 @@ def reducer_override(self, obj): # hard-coded call to save_global when pickling meta-classes. dispatch = Pickler.dispatch.copy() - def __init__(self, file, protocol=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - Pickler.__init__(self, file, protocol=protocol) - # map functions __globals__ attribute ids, to ensure that functions - # sharing the same global namespace at pickling time also share - # their global namespace at unpickling time. - self.globals_ref = {} - assert hasattr(self, 'proto') - - def _save_reduce_pickle5(self, func, args, state=None, listitems=None, - dictitems=None, state_setter=None, obj=None): + def _save_reduce_pickle5( + self, + func, + args, + state=None, + listitems=None, + dictitems=None, + state_setter=None, + obj=None, + ): save = self.save write = self.write self.save_reduce( - func, args, state=None, listitems=listitems, - dictitems=dictitems, obj=obj + func, + args, + state=None, + listitems=listitems, + dictitems=dictitems, + obj=obj, ) # backport of the Python 3.8 state_setter pickle operations save(state_setter) @@ -778,9 +900,12 @@ def save_global(self, obj, name=None, pack=struct.pack): return self.save_reduce(type, (NotImplemented,), obj=obj) elif obj in _BUILTIN_TYPE_NAMES: return self.save_reduce( - _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) + _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj + ) - if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch + if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint( + obj + ): # noqa # pragma: no branch # Parametrized typing constructs in Python < 3.7 are not # compatible with type checks and ``isinstance`` semantics. For # this reason, it is easier to detect them using a @@ -789,7 +914,7 @@ def save_global(self, obj, name=None, pack=struct.pack): self.save_reduce( _create_parametrized_type_hint, parametrized_type_hint_getinitargs(obj), - obj=obj + obj=obj, ) elif name is not None: Pickler.save_global(self, obj, name=name) @@ -797,10 +922,11 @@ def save_global(self, obj, name=None, pack=struct.pack): self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) else: Pickler.save_global(self, obj, name=name) + dispatch[type] = save_global def save_function(self, obj, name=None): - """ Registered with the dispatch to handle all function types. + """Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. @@ -831,9 +957,11 @@ def save_pypy_builtin_func(self, obj): this routing should be removed when cloudpickle supports only PyPy 3.6 and later. """ - rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, - obj.__defaults__, obj.__closure__), - obj.__dict__) + rv = ( + types.FunctionType, + (obj.__code__, {}, obj.__name__, obj.__defaults__, obj.__closure__), + obj.__dict__, + ) self.save_reduce(*rv, obj=obj) dispatch[types.FunctionType] = save_function diff --git a/python/requirements.txt b/python/requirements.txt index 395b12c4c7ef..e04cdcb60830 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -65,6 +65,7 @@ aiohttp_cors opentelemetry-api==1.1.0 pyyaml gpustat>=1.0.0 -pydantic +pydantic; python_version < '3.11' +pydantic==1.10.2; python_version >= '3.11' prometheus_client<0.14.0,>=0.7.1 opentelemetry-sdk==1.1.0 diff --git a/python/requirements_test.txt b/python/requirements_test.txt index 1bec78ea9f34..cd0442fb6081 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -30,7 +30,7 @@ google-cloud-storage==2.5.0 gradio==3.5; platform_system != "Windows" jsonpatch==1.32 kubernetes==24.2.0 -llvmlite==0.39.1 +llvmlite==0.39.1; python_version < '3.11' lxml==4.9.1 moto[s3,server]==4.0.7 mypy==0.982 @@ -43,11 +43,13 @@ opentelemetry-exporter-otlp==1.1.0 pexpect==4.8.0 Pillow==9.2.0; platform_system != "Windows" proxy.py==2.4.3 -pyarrow==6.0.1 -pydantic==1.9.2 +pyarrow==6.0.1; python_version < '3.11' +pyarrow==10.0.1; python_version >= '3.11' +pydantic==1.9.2; python_version < '3.11' +pydantic==1.10.2; python_version >= '3.11' # Keep in sync with `ci/build/upload_build_info.sh` PyOpenSSL==22.1.0 -pygame==2.1.2 +pygame==2.1.2; python_version < '3.11' Pygments==2.13.0 pymongo==4.3.2 pyspark==3.3.1 @@ -59,7 +61,7 @@ pytest-lazy-fixture==0.6.3 pytest-timeout==2.1.0 pytest-virtualenv==1.7.0 redis==3.5.3 -scikit-learn==1.0.2 +scikit-learn==1.0.2; python_version < '3.11' smart_open[s3]==6.2.0 tqdm==4.64.1 testfixtures==7.0.0 diff --git a/python/setup.py b/python/setup.py index fdc8e69a85e7..eb4872f9eb19 100644 --- a/python/setup.py +++ b/python/setup.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) -SUPPORTED_PYTHONS = [(3, 6), (3, 7), (3, 8), (3, 9), (3, 10)] +SUPPORTED_PYTHONS = [(3, 6), (3, 7), (3, 8), (3, 9), (3, 10), (3, 11)] # When the bazel version is updated, make sure to update it # in WORKSPACE file as well. @@ -775,7 +775,7 @@ def has_ext_modules(self): # The BinaryDistribution argument triggers build_ext. distclass=BinaryDistribution, install_requires=setup_spec.install_requires, - setup_requires=["cython >= 0.29.26", "wheel"], + setup_requires=["cython >= 0.29.32", "wheel"], extras_require=setup_spec.extras, entry_points={ "console_scripts": [