From ccbcc4bafa79c8de44f99d7d089c2e9640951150 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Fri, 8 Nov 2019 15:58:28 -0800 Subject: [PATCH] Use GRCP and Bazel 1.0 (#6002) --- .bazelrc | 6 +- BUILD.bazel | 7 +- WORKSPACE | 7 + bazel/BUILD.tpl | 36 ---- bazel/cython_library.bzl | 73 --------- bazel/python_configure.bzl | 299 ---------------------------------- bazel/ray_deps_build_all.bzl | 8 +- bazel/ray_deps_setup.bzl | 53 +++--- build.sh | 4 +- ci/suppress_output | 4 +- ci/travis/install-bazel.sh | 2 +- java/BUILD.bazel | 2 +- src/ray/rpc/client_call.h | 8 +- src/ray/rpc/server_call.h | 4 +- src/ray/test/run_gcs_tests.sh | 18 +- 15 files changed, 63 insertions(+), 468 deletions(-) delete mode 100644 bazel/BUILD.tpl delete mode 100644 bazel/cython_library.bzl delete mode 100644 bazel/python_configure.bzl diff --git a/.bazelrc b/.bazelrc index c3cfabd19b3f..1509623141b4 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,7 +1,8 @@ # build config build --compilation_mode=opt build --action_env=PATH -build --action_env=PYTHON_BIN_PATH +build --action_env=PYTHON2_BIN_PATH +build --action_env=PYTHON3_BIN_PATH # Warnings should be errors build --per_file_copt=-src/ray/thirdparty/hiredis/dict.c,-.*/arrow/util/logging.cc@-Werror # Ignore warnings for protobuf generated files and external projects. @@ -10,3 +11,6 @@ build --per_file_copt='external*@-w' # This workaround is needed due to https://github.com/bazelbuild/bazel/issues/4341 build --per_file_copt="external/com_github_grpc_grpc/.*@-DGRPC_BAZEL_BUILD" build --http_timeout_scaling=5.0 +# This workaround is due to an incompatibility of +# bazel_common/tools/maven/pom_file.bzl with Bazel 1.0 +build --incompatible_depset_is_not_iterable=false diff --git a/BUILD.bazel b/BUILD.bazel index 7e63f4416fd0..2ac2f7f2272b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2,11 +2,11 @@ # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") -load("@build_stack_rules_proto//python:python_proto_compile.bzl", "python_proto_compile") -load("@build_stack_rules_proto//python:python_grpc_compile.bzl", "python_grpc_compile") +load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library") +load("@rules_proto_grpc//python:defs.bzl", "python_proto_compile") +load("@rules_proto_grpc//python:defs.bzl", "python_grpc_compile") load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") load("@//bazel:ray.bzl", "flatbuffer_py_library") -load("@//bazel:cython_library.bzl", "pyx_library") COPTS = ["-DRAY_USE_GLOG"] + select({ "@bazel_tools//src/conditions:windows": [ @@ -744,7 +744,6 @@ pyx_library( "python/ray/includes/*.pxd", "python/ray/includes/*.pxi", ]), - copts = COPTS, deps = [ "//:core_worker_lib", "//:raylet_lib", diff --git a/WORKSPACE b/WORKSPACE index 392601a50a5a..35038a3a1fc0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -7,3 +7,10 @@ ray_deps_setup() load("//bazel:ray_deps_build_all.bzl", "ray_deps_build_all") ray_deps_build_all() + +# This needs to be run after grpc_deps() in ray_deps_build_all() to make +# sure all the packages loaded by grpc_deps() are available. However a +# load() statement cannot be in a function so we put it here. +load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") + +grpc_extra_deps() diff --git a/bazel/BUILD.tpl b/bazel/BUILD.tpl deleted file mode 100644 index 257045c2ee8f..000000000000 --- a/bazel/BUILD.tpl +++ /dev/null @@ -1,36 +0,0 @@ -# Adapted from grpc/third_party/py/BUILD.tpl - -# Adapted with modifications from tensorflow/third_party/py/ - -package(default_visibility=["//visibility:public"]) - -# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib -# See https://docs.python.org/3/extending/windows.html -cc_import( - name="python_lib", - interface_library=select({ - ":windows": ":python_import_lib", - # A placeholder for Unix platforms which makes --no_build happy. - "//conditions:default": "not-existing.lib", - }), - system_provided=1, -) - -cc_library( - name="python_headers", - hdrs=[":python_include"], - deps=select({ - ":windows": [":python_lib"], - "//conditions:default": [], - }), - includes=["python_include"], -) - -config_setting( - name="windows", - values={"cpu": "x64_windows"}, - visibility=["//visibility:public"], -) - -%{PYTHON_INCLUDE_GENRULE} -%{PYTHON_IMPORT_LIB_GENRULE} diff --git a/bazel/cython_library.bzl b/bazel/cython_library.bzl deleted file mode 100644 index 115e5595f1e2..000000000000 --- a/bazel/cython_library.bzl +++ /dev/null @@ -1,73 +0,0 @@ -"""Custom rules for gRPC Python""" - -# Adapted from grpc/bazel/cython_library.bzl - -# Adapted with modifications from -# tensorflow/tensorflow/core/platform/default/build_config.bzl -# Native Bazel rules don't exist yet to compile Cython code, but rules have -# been written at cython/cython and tensorflow/tensorflow. We branch from -# Tensorflow's version as it is more actively maintained and works for gRPC -# Python's needs. -def pyx_library(name, deps=[], py_deps=[], srcs=[], copts=[], **kwargs): - """Compiles a group of .pyx / .pxd / .py files. - First runs Cython to create .cpp files for each input .pyx or .py + .pxd - pair. Then builds a shared object for each, passing "deps" to each cc_binary - rule (includes Python headers by default). Finally, creates a py_library rule - with the shared objects and any pure Python "srcs", with py_deps as its - dependencies; the shared objects can be imported like normal Python files. - Args: - name: Name for the rule. - deps: C/C++ dependencies of the Cython (e.g. Numpy headers). - py_deps: Pure Python dependencies of the final library. - srcs: .py, .pyx, or .pxd files to either compile or pass through. - **kwargs: Extra keyword arguments passed to the py_library. - """ - # First filter out files that should be run compiled vs. passed through. - py_srcs = [] - pyx_srcs = [] - pxd_srcs = [] - for src in srcs: - if src.endswith(".pyx") or (src.endswith(".py") and - src[:-3] + ".pxd" in srcs): - pyx_srcs.append(src) - elif src.endswith(".py"): - py_srcs.append(src) - else: - pxd_srcs.append(src) - if src.endswith("__init__.py"): - pxd_srcs.append(src) - - # Invoke cython to produce the shared object libraries. - for filename in pyx_srcs: - native.genrule( - name=filename + "_cython_translation", - srcs=[filename], - outs=[filename.split(".")[0] + ".cpp"], - # Optionally use PYTHON_BIN_PATH on Linux platforms so that python 3 - # works. Windows has issues with cython_binary so skip PYTHON_BIN_PATH. - cmd= - 'PYTHONHASHSEED=0 "$${PYTHON_BIN_PATH}" "$(location @cython//:cython_binary)" --cplus $(SRCS) --output-file $(OUTS)', - tools=["@cython//:cython_binary"] + pxd_srcs, - ) - - shared_objects = [] - for src in pyx_srcs: - stem = src.split(".")[0] - shared_object_name = stem + ".so" - native.cc_binary( - name=shared_object_name, - srcs=[stem + ".cpp"], - copts=copts, - deps=deps + ["@local_config_python//:python_headers"], - linkshared=1, - ) - shared_objects.append(shared_object_name) - - # Now create a py_library with these shared objects as data. - native.py_library( - name=name, - srcs=py_srcs, - deps=py_deps, - srcs_version="PY2AND3", - data=shared_objects, - **kwargs) diff --git a/bazel/python_configure.bzl b/bazel/python_configure.bzl deleted file mode 100644 index 5dec11e50f93..000000000000 --- a/bazel/python_configure.bzl +++ /dev/null @@ -1,299 +0,0 @@ -# Adapted with a minor modification from grpc/third_party/py/python_configure.bzl: -# Changed "//third_party/py:%s.tpl" -> "@//bazel:%s.tpl" - -# Adapted with modifications from tensorflow/third_party/py/ - -"""Repository rule for Python autoconfiguration. -`python_configure` depends on the following environment variables: - * `PYTHON_BIN_PATH`: location of python binary. - * `PYTHON_LIB_PATH`: Location of python libraries. -""" - -_BAZEL_SH = "BAZEL_SH" -_PYTHON_BIN_PATH = "PYTHON_BIN_PATH" -_PYTHON_LIB_PATH = "PYTHON_LIB_PATH" -_PYTHON_CONFIG_REPO = "PYTHON_CONFIG_REPO" - - -def _tpl(repository_ctx, tpl, substitutions={}, out=None): - if not out: - out = tpl - repository_ctx.template(out, Label("@//bazel:%s.tpl" % tpl), - substitutions) - - -def _fail(msg): - """Output failure message when auto configuration fails.""" - red = "\033[0;31m" - no_color = "\033[0m" - fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) - - -def _is_windows(repository_ctx): - """Returns true if the host operating system is windows.""" - os_name = repository_ctx.os.name.lower() - return os_name.find("windows") != -1 - - -def _execute(repository_ctx, - cmdline, - error_msg=None, - error_details=None, - empty_stdout_fine=False): - """Executes an arbitrary shell command. - Args: - repository_ctx: the repository_ctx object - cmdline: list of strings, the command to execute - error_msg: string, a summary of the error if the command fails - error_details: string, details about the error or steps to fix it - empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise - it's an error - Return: - the result of repository_ctx.execute(cmdline) - """ - result = repository_ctx.execute(cmdline) - if result.stderr or not (empty_stdout_fine or result.stdout): - _fail("\n".join([ - error_msg.strip() if error_msg else "Repository command failed", - result.stderr.strip(), error_details if error_details else "" - ])) - else: - return result - - -def _read_dir(repository_ctx, src_dir): - """Returns a string with all files in a directory. - Finds all files inside a directory, traversing subfolders and following - symlinks. The returned string contains the full path of all files - separated by line breaks. - """ - if _is_windows(repository_ctx): - src_dir = src_dir.replace("/", "\\") - find_result = _execute( - repository_ctx, - ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], - empty_stdout_fine=True) - # src_files will be used in genrule.outs where the paths must - # use forward slashes. - return find_result.stdout.replace("\\", "/") - else: - find_result = _execute( - repository_ctx, ["find", src_dir, "-follow", "-type", "f"], - empty_stdout_fine=True) - return find_result.stdout - - -def _genrule(src_dir, genrule_name, command, outs): - """Returns a string with a genrule. - Genrule executes the given command and produces the given outputs. - """ - return ('genrule(\n' + ' name = "' + genrule_name + '",\n' + - ' outs = [\n' + outs + '\n ],\n' + ' cmd = """\n' + - command + '\n """,\n' + ')\n') - - -def _normalize_path(path): - """Returns a path with '/' and remove the trailing slash.""" - path = path.replace("\\", "/") - if path[-1] == "/": - path = path[:-1] - return path - - -def _symlink_genrule_for_dir(repository_ctx, - src_dir, - dest_dir, - genrule_name, - src_files=[], - dest_files=[]): - """Returns a genrule to symlink(or copy if on Windows) a set of files. - If src_dir is passed, files will be read from the given directory; otherwise - we assume files are in src_files and dest_files - """ - if src_dir != None: - src_dir = _normalize_path(src_dir) - dest_dir = _normalize_path(dest_dir) - files = '\n'.join( - sorted(_read_dir(repository_ctx, src_dir).splitlines())) - # Create a list with the src_dir stripped to use for outputs. - dest_files = files.replace(src_dir, '').splitlines() - src_files = files.splitlines() - command = [] - outs = [] - for i in range(len(dest_files)): - if dest_files[i] != "": - # If we have only one file to link we do not want to use the dest_dir, as - # $(@D) will include the full path to the file. - dest = '$(@D)/' + dest_dir + dest_files[i] if len( - dest_files) != 1 else '$(@D)/' + dest_files[i] - # On Windows, symlink is not supported, so we just copy all the files. - cmd = 'cp -f' if _is_windows(repository_ctx) else 'ln -s' - command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) - outs.append(' "' + dest_dir + dest_files[i] + '",') - return _genrule(src_dir, genrule_name, " && ".join(command), - "\n".join(outs)) - - -def _get_python_bin(repository_ctx): - """Gets the python bin path.""" - python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) - if python_bin != None: - return python_bin - python_bin_path = repository_ctx.which("python") - if python_bin_path != None: - return str(python_bin_path) - _fail("Cannot find python in PATH, please make sure " + - "python is installed and add its directory in PATH, or --define " + - "%s='/something/else'.\nPATH=%s" % - (_PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) - - -def _get_bash_bin(repository_ctx): - """Gets the bash bin path.""" - bash_bin = repository_ctx.os.environ.get(_BAZEL_SH) - if bash_bin != None: - return bash_bin - else: - bash_bin_path = repository_ctx.which("bash") - if bash_bin_path != None: - return str(bash_bin_path) - else: - _fail( - "Cannot find bash in PATH, please make sure " + - "bash is installed and add its directory in PATH, or --define " - + "%s='/path/to/bash'.\nPATH=%s" % - (_BAZEL_SH, repository_ctx.os.environ.get("PATH", ""))) - - -def _get_python_lib(repository_ctx, python_bin): - """Gets the python lib path.""" - python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) - if python_lib != None: - return python_lib - print_lib = ( - "<=1:\n" + - " print(paths[0])\n" + "END") - cmd = '"%s" - %s' % (python_bin, print_lib) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) - return result.stdout.strip('\n') - - -def _check_python_lib(repository_ctx, python_lib): - """Checks the python lib path.""" - cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) - if result.return_code == 1: - _fail("Invalid python library path: %s" % python_lib) - - -def _check_python_bin(repository_ctx, python_bin): - """Checks the python bin path.""" - cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) - result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) - if result.return_code == 1: - _fail("--define %s='%s' is not executable. Is it the python binary?" % - (_PYTHON_BIN_PATH, python_bin)) - - -def _get_python_include(repository_ctx, python_bin): - """Gets the python include path.""" - result = _execute( - repository_ctx, [ - python_bin, "-c", 'from __future__ import print_function;' + - 'from distutils import sysconfig;' + - 'print(sysconfig.get_python_inc())' - ], - error_msg="Problem getting python include path.", - error_details=( - "Is the Python binary path set up right? " + "(See ./configure or " - + _PYTHON_BIN_PATH + ".) " + "Is distutils installed?")) - return result.stdout.splitlines()[0] - - -def _get_python_import_lib_name(repository_ctx, python_bin): - """Get Python import library name (pythonXY.lib) on Windows.""" - result = _execute( - repository_ctx, [ - python_bin, "-c", - 'import sys;' + 'print("python" + str(sys.version_info[0]) + ' + - ' str(sys.version_info[1]) + ".lib")' - ], - error_msg="Problem getting python import library.", - error_details=("Is the Python binary path set up right? " + - "(See ./configure or " + _PYTHON_BIN_PATH + ".) ")) - return result.stdout.splitlines()[0] - - -def _create_local_python_repository(repository_ctx): - """Creates the repository containing files set up to build with Python.""" - python_bin = _get_python_bin(repository_ctx) - _check_python_bin(repository_ctx, python_bin) - python_lib = _get_python_lib(repository_ctx, python_bin) - _check_python_lib(repository_ctx, python_lib) - python_include = _get_python_include(repository_ctx, python_bin) - python_include_rule = _symlink_genrule_for_dir( - repository_ctx, python_include, 'python_include', 'python_include') - python_import_lib_genrule = "" - # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib - # See https://docs.python.org/3/extending/windows.html - if _is_windows(repository_ctx): - python_include = _normalize_path(python_include) - python_import_lib_name = _get_python_import_lib_name( - repository_ctx, python_bin) - python_import_lib_src = python_include.rsplit( - '/', 1)[0] + "/libs/" + python_import_lib_name - python_import_lib_genrule = _symlink_genrule_for_dir( - repository_ctx, None, '', 'python_import_lib', - [python_import_lib_src], [python_import_lib_name]) - _tpl( - repository_ctx, "BUILD", { - "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, - "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, - }) - - -def _create_remote_python_repository(repository_ctx, remote_config_repo): - """Creates pointers to a remotely configured repo set up to build with Python. - """ - _tpl(repository_ctx, "remote.BUILD", { - "%{REMOTE_PYTHON_REPO}": remote_config_repo, - }, "BUILD") - - -def _python_autoconf_impl(repository_ctx): - """Implementation of the python_autoconf repository rule.""" - if _PYTHON_CONFIG_REPO in repository_ctx.os.environ: - _create_remote_python_repository( - repository_ctx, repository_ctx.os.environ[_PYTHON_CONFIG_REPO]) - else: - _create_local_python_repository(repository_ctx) - - -python_configure = repository_rule( - implementation=_python_autoconf_impl, - environ=[ - _BAZEL_SH, - _PYTHON_BIN_PATH, - _PYTHON_LIB_PATH, - _PYTHON_CONFIG_REPO, - ], -) -"""Detects and configures the local Python. -Add the following to your WORKSPACE FILE: -```python -python_configure(name = "local_config_python") -``` -Args: - name: A unique name for this workspace rule. -""" diff --git a/bazel/ray_deps_build_all.bzl b/bazel/ray_deps_build_all.bzl index 541661a3f22f..8794773d911d 100644 --- a/bazel/ray_deps_build_all.bzl +++ b/bazel/ray_deps_build_all.bzl @@ -1,11 +1,10 @@ load("@com_github_ray_project_ray//java:dependencies.bzl", "gen_java_deps") load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps") load("@com_github_jupp0r_prometheus_cpp//:repositories.bzl", "prometheus_cpp_repositories") -load("@com_github_ray_project_ray//bazel:python_configure.bzl", "python_configure") load("@com_github_checkstyle_java//:repo.bzl", "checkstyle_deps") +load("@com_github_grpc_grpc//third_party/py:python_configure.bzl", "python_configure") load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") -load("@build_stack_rules_proto//java:deps.bzl", "java_proto_compile") -load("@build_stack_rules_proto//python:deps.bzl", "python_proto_compile") +load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains") def ray_deps_build_all(): @@ -15,5 +14,4 @@ def ray_deps_build_all(): prometheus_cpp_repositories() python_configure(name = "local_config_python") grpc_deps() - java_proto_compile() - python_proto_compile() + rules_proto_grpc_toolchains() diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl index 42bb097dd9ba..a81e07db7a70 100644 --- a/bazel/ray_deps_setup.bzl +++ b/bazel/ray_deps_setup.bzl @@ -2,9 +2,8 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_r load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def ray_deps_setup(): - RULES_JVM_EXTERNAL_TAG = "1.2" - - RULES_JVM_EXTERNAL_SHA = "e5c68b87f750309a79f59c2b69ead5c3221ffa54ff9496306937bfa1c9c8c86b" + RULES_JVM_EXTERNAL_TAG = "2.10" + RULES_JVM_EXTERNAL_SHA = "1bbf2e48d07686707dd85357e9a94da775e1dbd7c464272b3664283c9c716d26" http_archive( name = "rules_jvm_external", @@ -17,7 +16,7 @@ def ray_deps_setup(): name = "bazel_common", strip_prefix = "bazel-common-f1115e0f777f08c3cdb115526c4e663005bec69b", url = "https://github.com/google/bazel-common/archive/f1115e0f777f08c3cdb115526c4e663005bec69b.zip", - sha256 = "1e05a4791cc3470d3ecf7edb556f796b1d340359f1c4d293f175d4d0946cf84c", + sha256 = "1e05a4791cc3470d3ecf7edb556f796b1d340359f1c4d293f175d4d0946cf84c", ) BAZEL_SKYLIB_TAG = "0.6.0" @@ -26,49 +25,50 @@ def ray_deps_setup(): name = "bazel_skylib", strip_prefix = "bazel-skylib-%s" % BAZEL_SKYLIB_TAG, url = "https://github.com/bazelbuild/bazel-skylib/archive/%s.tar.gz" % BAZEL_SKYLIB_TAG, - sha256 = "eb5c57e4c12e68c0c20bc774bfbc60a568e800d025557bc4ea022c6479acc867", + sha256 = "eb5c57e4c12e68c0c20bc774bfbc60a568e800d025557bc4ea022c6479acc867", ) git_repository( name = "com_github_checkstyle_java", - commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b", - remote = "https://github.com/ruifangChen/checkstyle_java", - shallow_since = "1552542575 +0800", + commit = "ef367030d1433877a3360bbfceca18a5d0791bdd", + remote = "https://github.com/ray-project/checkstyle_java", + shallow_since = "1573090990 -0800", ) git_repository( name = "com_github_nelhage_rules_boost", commit = "5171b9724fbb39c5fdad37b9ca9b544e8858d8ac", remote = "https://github.com/ray-project/rules_boost", - shallow_since = "1556014830 +0800", + shallow_since = "1556014830 +0800", ) git_repository( name = "com_github_google_flatbuffers", commit = "63d51afd1196336a7d1f56a988091ef05deb1c62", remote = "https://github.com/google/flatbuffers.git", - shallow_since = "1547755012 -0800", + shallow_since = "1547755012 -0800", ) git_repository( name = "com_google_googletest", commit = "3306848f697568aacf4bcca330f6bdd5ce671899", remote = "https://github.com/google/googletest", - shallow_since = "1534270723 -0700", + shallow_since = "1534270723 -0700", ) git_repository( name = "com_github_gflags_gflags", remote = "https://github.com/gflags/gflags.git", - commit = "e171aa2d15ed9eb17054558e0b3a6a413bb01067", - shallow_since = "1541971260 +0000", + commit = "e171aa2d15ed9eb17054558e0b3a6a413bb01067", + shallow_since = "1541971260 +0000", ) new_git_repository( name = "com_github_google_glog", build_file = "@//bazel:BUILD.glog", - commit = "5c576f78c49b28d89b23fbb1fc80f54c879ec02e", + commit = "96a2f23dca4cc7180821ca5f32e526314395d26a", remote = "https://github.com/google/glog", + shallow_since = "1550458164 +0900", ) new_git_repository( @@ -76,6 +76,7 @@ def ray_deps_setup(): build_file = "@//bazel:BUILD.plasma", commit = "86f34aa07e611787d9cc98c6a33b0a0a536dce57", remote = "https://github.com/apache/arrow", + shallow_since = "1572492886 -0700", ) new_git_repository( @@ -83,14 +84,14 @@ def ray_deps_setup(): build_file = "@//bazel:BUILD.cython", commit = "49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e", remote = "https://github.com/cython/cython", - shallow_since = "1547888711 +0100", + shallow_since = "1547888711 +0100", ) http_archive( name = "io_opencensus_cpp", strip_prefix = "opencensus-cpp-3aa11f20dd610cb8d2f7c62e58d1e69196aadf11", urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/3aa11f20dd610cb8d2f7c62e58d1e69196aadf11.zip"], - sha256 = "92eef77c44d01e8472f68a2f1329919a1bb59317a4bb1e4d76081ab5c13a56d6", + sha256 = "92eef77c44d01e8472f68a2f1329919a1bb59317a4bb1e4d76081ab5c13a56d6", ) # OpenCensus depends on Abseil so we have to explicitly pull it in. @@ -99,7 +100,7 @@ def ray_deps_setup(): name = "com_google_absl", commit = "aa844899c937bde5d2b24f276b59997e5b668bde", remote = "https://github.com/abseil/abseil-cpp.git", - shallow_since = "1565288385 -0400", + shallow_since = "1565288385 -0400", ) # OpenCensus depends on jupp0r/prometheus-cpp @@ -109,19 +110,19 @@ def ray_deps_setup(): # TODO(qwang): We should use the repository of `jupp0r` here when this PR # `https://github.com/jupp0r/prometheus-cpp/pull/225` getting merged. remote = "https://github.com/ray-project/prometheus-cpp.git", + shallow_since = "1572744904 -0700" ) - http_archive( + git_repository( name = "com_github_grpc_grpc", - urls = [ - "https://github.com/grpc/grpc/archive/76a381869413834692b8ed305fbe923c0f9c4472.tar.gz", - ], - strip_prefix = "grpc-76a381869413834692b8ed305fbe923c0f9c4472", - sha256 = "b5efbe086b9a00826a3f830094312e6d1647157b5a5e7954a8ac4179bce3de8b", + commit = "93e8830070e9afcbaa992c75817009ee3f4b63a0", + remote = "https://github.com/grpc/grpc.git", + shallow_since = "1571118670 -0700", ) git_repository( - name = "build_stack_rules_proto", - remote = "https://github.com/stackb/rules_proto.git", - commit = "d9a123032f8436dbc34069cfc3207f2810a494ee", + name = "rules_proto_grpc", + commit = "a74fef39c5fe636580083545f76d1eab74f6450d", + remote = "https://github.com/rules-proto-grpc/rules_proto_grpc.git", + shallow_since = "1571494564 +0100", ) diff --git a/build.sh b/build.sh index 89c6893c62e4..c55a4763e7c8 100755 --- a/build.sh +++ b/build.sh @@ -101,7 +101,9 @@ if [ -z "$SKIP_PYARROW_INSTALL" ]; then --target="$ROOT_DIR/python/ray/pyarrow_files" pyarrow==0.14.0.RAY \ --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/3a11193d9530fe8ec7fdb98057f853b708f6f6ae/index.html fi -export PYTHON_BIN_PATH="$PYTHON_EXECUTABLE" + +export PYTHON3_BIN_PATH="$PYTHON_EXECUTABLE" +export PYTHON2_BIN_PATH="$PYTHON_EXECUTABLE" if [ "$RAY_BUILD_JAVA" == "YES" ]; then "$BAZEL_EXECUTABLE" build //java:all --verbose_failures diff --git a/ci/suppress_output b/ci/suppress_output index 0f32b1a88b37..583bd6275e54 100755 --- a/ci/suppress_output +++ b/ci/suppress_output @@ -6,11 +6,11 @@ PID=$$ # Print output to avoid travis killing us watchdog() { - for i in `seq 5 5 120`; do + for i in `seq 5 5 150`; do sleep 300 echo "This command has been running for more than $i minutes..." done - echo "Command timed out after 2h, dumping logs:" + echo "Command timed out after 2.5h, dumping logs:" cat $TMPFILE echo "TIMED OUT" kill -SIGKILL $PID diff --git a/ci/travis/install-bazel.sh b/ci/travis/install-bazel.sh index e9e7e9f906b6..0627bae6c553 100755 --- a/ci/travis/install-bazel.sh +++ b/ci/travis/install-bazel.sh @@ -16,7 +16,7 @@ else exit 1 fi -URL="https://github.com/bazelbuild/bazel/releases/download/0.26.1/bazel-0.26.1-installer-${platform}-x86_64.sh" +URL="https://github.com/bazelbuild/bazel/releases/download/1.1.0/bazel-1.1.0-installer-${platform}-x86_64.sh" wget -O install.sh $URL chmod +x install.sh ./install.sh --user diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 37ef5b93b6ee..1570d763c42e 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -1,5 +1,5 @@ load("//bazel:ray.bzl", "define_java_module") -load("@build_stack_rules_proto//java:java_proto_compile.bzl", "java_proto_compile") +load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile") exports_files([ "testng.xml", diff --git a/src/ray/rpc/client_call.h b/src/ray/rpc/client_call.h index fe1462535dd3..72384ca3fcda 100644 --- a/src/ray/rpc/client_call.h +++ b/src/ray/rpc/client_call.h @@ -62,7 +62,7 @@ class ClientCallImpl : public ClientCall { ClientCallback callback_; /// The response reader. - std::unique_ptr> response_reader_; + std::unique_ptr> response_reader_; /// gRPC status of this request. grpc::Status status_; @@ -106,9 +106,9 @@ class ClientCallTag { /// \tparam Request Type of the request message. /// \tparam Reply Type of the reply message. template -using PrepareAsyncFunction = std::unique_ptr> ( - GrpcService::Stub::*)(grpc::ClientContext *context, const Request &request, - grpc::CompletionQueue *cq); +using PrepareAsyncFunction = + std::unique_ptr> (GrpcService::Stub::*)( + grpc::ClientContext *context, const Request &request, grpc::CompletionQueue *cq); /// `ClientCallManager` is used to manage outgoing gRPC requests and the lifecycles of /// `ClientCall` objects. diff --git a/src/ray/rpc/server_call.h b/src/ray/rpc/server_call.h index 87fab50192c8..49fe28dc61fe 100644 --- a/src/ray/rpc/server_call.h +++ b/src/ray/rpc/server_call.h @@ -198,7 +198,7 @@ class ServerCallImpl : public ServerCall { grpc::ServerContext context_; /// The response writer. - grpc::ServerAsyncResponseWriter response_writer_; + grpc_impl::ServerAsyncResponseWriter response_writer_; /// The event loop. boost::asio::io_service &io_service_; @@ -226,7 +226,7 @@ class ServerCallImpl : public ServerCall { /// \tparam Reply Type of the reply message. template using RequestCallFunction = void (GrpcService::AsyncService::*)( - grpc::ServerContext *, Request *, grpc::ServerAsyncResponseWriter *, + grpc::ServerContext *, Request *, grpc_impl::ServerAsyncResponseWriter *, grpc::CompletionQueue *, grpc::ServerCompletionQueue *, void *); /// Implementation of `ServerCallFactory` diff --git a/src/ray/test/run_gcs_tests.sh b/src/ray/test/run_gcs_tests.sh index d4c86f90f13b..969bdc44a008 100644 --- a/src/ray/test/run_gcs_tests.sh +++ b/src/ray/test/run_gcs_tests.sh @@ -9,18 +9,10 @@ set -x bazel build "//:redis_gcs_client_test" "//:actor_state_accessor_test" "//:subscription_executor_test" "//:asio_test" "//:libray_redis_module.so" # Start Redis. -if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then - ./src/credis/redis/src/redis-server \ - --loglevel warning \ - --loadmodule ./src/credis/build/src/libmember.so \ - --loadmodule ./src/ray/gcs/redis_module/libray_redis_module.so \ - --port 6379 & -else - ./bazel-genfiles/redis-server \ - --loglevel warning \ - --loadmodule ./bazel-bin/libray_redis_module.so \ - --port 6379 & -fi +./bazel-bin/redis-server \ + --loglevel warning \ + --loadmodule ./bazel-bin/libray_redis_module.so \ + --port 6379 & sleep 1s ./bazel-bin/redis_gcs_client_test @@ -28,5 +20,5 @@ sleep 1s ./bazel-bin/subscription_executor_test ./bazel-bin/asio_test -./bazel-genfiles/redis-cli -p 6379 shutdown +./bazel-bin/redis-cli -p 6379 shutdown sleep 1s