Revert "Merge remote-tracking branch 'upstream/master' into gce-infer…

…ence" This reverts commit 7ae61ee, reversing changes made to 6cb7bb0. Signed-off-by: Balaji Veeramani <[email protected]>
bveeramani · Apr 5, 2023 · f88b44d · f88b44d
1 parent 7ae61ee
commit f88b44d
Show file tree

Hide file tree

Showing 172 changed files with 1,032 additions and 4,416 deletions.
diff --git a/ci/lint/check_api_annotations.py b/ci/lint/check_api_annotations.py
@@ -99,13 +99,7 @@ def verify(symbol, scanned, ok, output, prefix=None, ignore=None):
     verify(ray.air, set(), ok, output)
     verify(ray.train, set(), ok, output)
     verify(ray.tune, set(), ok, output)
-    verify(
-        ray,
-        set(),
-        ok,
-        output,
-        ignore=["ray.workflow", "ray.tune", "ray.serve"],
-    )
+    verify(ray, set(), ok, output, ignore=["ray.workflow", "ray.tune", "ray.serve"])
     verify(ray.serve, set(), ok, output)
     assert len(ok) >= 500, len(ok)
     # TODO(ekl) enable it for all modules.

diff --git a/dashboard/agent.py b/dashboard/agent.py
@@ -5,7 +5,6 @@
 import logging
 import logging.handlers
 import os
-import pathlib
 import sys
 import signal
 
@@ -18,10 +17,7 @@
 from ray.dashboard.consts import _PARENT_DEATH_THREASHOLD
 from ray._private.gcs_pubsub import GcsAioPublisher, GcsPublisher
 from ray._private.gcs_utils import GcsAioClient, GcsClient
-from ray._private.ray_logging import (
-    setup_component_logger,
-    configure_log_file,
-)
+from ray._private.ray_logging import setup_component_logger
 from ray.core.generated import agent_manager_pb2, agent_manager_pb2_grpc
 from ray.experimental.internal_kv import (
     _initialize_internal_kv,
@@ -342,14 +338,6 @@ async def _check_parent():
             await self.http_server.cleanup()
 
 
-def open_capture_files(log_dir):
-    filename = f"agent-{args.agent_id}"
-    return (
-        ray._private.utils.open_log(pathlib.Path(log_dir) / f"{filename}.out"),
-        ray._private.utils.open_log(pathlib.Path(log_dir) / f"{filename}.err"),
-    )
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Dashboard agent.")
     parser.add_argument(
@@ -516,10 +504,6 @@ def open_capture_files(log_dir):
         # w.r.t grpc server init in the DashboardAgent initializer.
         loop = ray._private.utils.get_or_create_event_loop()
 
-        # Setup stdout/stderr redirect files
-        out_file, err_file = open_capture_files(args.log_dir)
-        configure_log_file(out_file, err_file)
-
         agent = DashboardAgent(
             args.node_ip_address,
             args.dashboard_agent_port,

diff --git a/dashboard/modules/job/job_manager.py b/dashboard/modules/job/job_manager.py
@@ -632,6 +632,21 @@ async def _monitor_job_internal(
         if job_supervisor is not None:
             ray.kill(job_supervisor, no_restart=True)
 
+    def _get_current_node_resource_key(self) -> str:
+        """Get the Ray resource key for current node.
+
+        It can be used for actor placement.
+        """
+        current_node_id = ray.get_runtime_context().get_node_id()
+        for node in ray.nodes():
+            if node["NodeID"] == current_node_id:
+                # Found the node.
+                for key in node["Resources"].keys():
+                    if key.startswith("node:"):
+                        return key
+        else:
+            raise ValueError("Cannot find the node dictionary for current node.")
+
     def _handle_supervisor_startup(self, job_id: str, result: Optional[Exception]):
         """Handle the result of starting a job supervisor actor.
 

diff --git a/dashboard/modules/log/log_head.py b/dashboard/modules/log/log_head.py
@@ -78,8 +78,8 @@ async def get_log_from_proxy(self, req) -> aiohttp.web.StreamResponse:
             )
             sr.content_length = r.content_length
             sr.content_type = r.content_type
-            if r.charset and not sr.content_type.startswith("application/octet-stream"):
-                sr.charset = r.charset
+            sr.charset = r.charset
+
             writer = await sr.prepare(req)
             async for data in r.content.iter_any():
                 await writer.write(data)

diff --git a/dashboard/modules/log/tests/test_log.py b/dashboard/modules/log/tests/test_log.py
@@ -135,66 +135,52 @@ def write_log(s):
 
 
 @pytest.mark.parametrize(
-    "test_file,content_kind",
-    [
-        ("test.log", "text/plain"),
-        ("test#1234.log", "text/plain"),
-        ("test_file_no_suffix", "application/octet-stream"),
-        ("test_file_not_register_mimetypes.json", "application/json"),
-        ("test_file_not_register_mimetypes.yaml", "application/octet-stream"),
-    ],
+    "test_file",
+    ["test.log", "test#1234.log"],
 )
-def test_log_proxy(ray_start_with_dashboard, test_file, content_kind):
+def test_log_proxy(ray_start_with_dashboard, test_file):
     assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True
     webui_url = ray_start_with_dashboard["webui_url"]
     webui_url = format_web_url(webui_url)
 
-    if content_kind == "text/plain":
-        data = bytearray("test_log_text", encoding="utf-8")
-    else:
-        data = bytearray(i for i in range(256))
-
-    # Prep the files
+    timeout_seconds = 5
+    start_time = time.time()
+    last_ex = None
+    test_log_text = "test_log_text"
     with open(
-        f"{ray._private.worker.global_worker.node.get_logs_dir_path()}/{test_file}",
-        "wb",
+        f"{ray._private.worker.global_worker.node.get_logs_dir_path()}/{test_file}", "w"
     ) as f:
-        f.write(data)
-
-    # Test basic fetching
-    def verify():
-        url = urllib.parse.quote(f"{webui_url}/logs/{test_file}")
-        response = requests.get(f"{webui_url}/log_proxy?url={url}")
-        response.raise_for_status()
-        assert response.content == data
-        return True
-
-    wait_for_condition(verify)
-
-    def verify():
-        url = urllib.parse.quote(f"{webui_url}/logs/{test_file}")
-        # Test range request.
-        response = requests.get(
-            f"{webui_url}/log_proxy?url={url}",
-            headers={
-                "Range": "bytes=2-5",
-            },
-        )
-        response.raise_for_status()
-        assert response.content == data[2:6]
-        return True
-
-    wait_for_condition(verify)
-
-    # Test 404.
-    def verify():
-        response = requests.get(
-            f"{webui_url}/log_proxy?" f"url={webui_url}/logs/not_exist_file.log"
-        )
-        assert response.status_code == 404
-        return True
-
-    wait_for_condition(verify)
+        f.write(test_log_text)
+    while True:
+        time.sleep(1)
+        try:
+            url = urllib.parse.quote(f"{webui_url}/logs/{test_file}")
+            # Test range request.
+            response = requests.get(
+                f"{webui_url}/log_proxy?url={url}",
+                headers={"Range": "bytes=2-5"},
+            )
+            response.raise_for_status()
+            assert response.text == test_log_text[2:6]
+            # Test 404.
+            response = requests.get(
+                f"{webui_url}/log_proxy?" f"url={webui_url}/logs/not_exist_file.log"
+            )
+            assert response.status_code == 404
+            break
+        except Exception as ex:
+            last_ex = ex
+        finally:
+            if time.time() > start_time + timeout_seconds:
+                ex_stack = (
+                    traceback.format_exception(
+                        type(last_ex), last_ex, last_ex.__traceback__
+                    )
+                    if last_ex
+                    else []
+                )
+                ex_stack = "".join(ex_stack)
+                raise Exception(f"Timed out while testing, {ex_stack}")
 
 
 @pytest.mark.parametrize(

diff --git a/dashboard/modules/state/state_head.py b/dashboard/modules/state/state_head.py
@@ -385,12 +385,7 @@ async def list_logs(self, req: aiohttp.web.Request) -> aiohttp.web.Response:
                 node_id, timeout, glob_filter=glob_filter
             )
         except DataSourceUnavailable as e:
-            return self._reply(
-                success=False,
-                error_message=str(e),
-                result=None,
-                reason=str(e),
-            )
+            return self._reply(success=False, error_message=str(e), result=None)
 
         return self._reply(success=True, error_message="", result=result)
 

diff --git a/dashboard/optional_utils.py b/dashboard/optional_utils.py
@@ -151,7 +151,7 @@ def predicate(o):
 
 
 def rest_response(
-    success, message, convert_google_style=True, reason=None, **kwargs
+    success, message, convert_google_style=True, **kwargs
 ) -> aiohttp.web.Response:
     # In the dev context we allow a dev server running on a
     # different port to consume the API, meaning we need to allow
@@ -169,7 +169,6 @@ def rest_response(
         dumps=functools.partial(json.dumps, cls=CustomEncoder),
         headers=headers,
         status=200 if success else 500,
-        reason=reason,
     )
 
 

diff --git a/doc/BUILD b/doc/BUILD
@@ -131,7 +131,6 @@ py_test_run_all_subdirectory(
     exclude = [
         "source/serve/doc_code/distilbert.py",
         "source/serve/doc_code/stable_diffusion.py",
-         "source/serve/doc_code/object_detection.py",
     ],
     extra_srcs = [],
     tags = ["exclusive", "team:serve"],
@@ -142,7 +141,6 @@ py_test_run_all_subdirectory(
     include = [
         "source/serve/doc_code/distilbert.py",
         "source/serve/doc_code/stable_diffusion.py",
-        "source/serve/doc_code/object_detection.py",
     ],
     exclude = [],
     extra_srcs = [],

diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt
@@ -61,10 +61,11 @@ sphinx-book-theme==0.3.3
 sphinx-external-toc==0.2.3
 sphinxcontrib.yt==0.2.2
 sphinx-sitemap==2.2.0
+sphinx-thebe==0.1.1
+autodoc_pydantic==1.6.1
 sphinxcontrib-redoc==1.6.0
 sphinx-tabs==3.4.0
 sphinx-remove-toctrees==0.0.3
-autodoc_pydantic==1.6.1
 
 # MyST
 myst-parser==0.15.2

diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml
@@ -35,7 +35,6 @@ parts:
               - file: ray-core/examples/batch_training
               - file: ray-core/examples/automl_for_time_series
               - file: ray-core/examples/web-crawler
-              - file: ray-core/examples/map_reduce
           - file: ray-core/api/index
 
       - file: ray-air/getting-started
@@ -382,7 +381,6 @@ parts:
         title: References
         sections:
           - file: cluster/usage-stats
-          - file: ray-references/glossary
 
       - file: ray-contribute/stability
         title: Developer Guides

diff --git a/doc/source/conf.py b/doc/source/conf.py
@@ -54,6 +54,7 @@
     "sphinx.ext.coverage",
     "sphinx.ext.autosummary",
     "sphinx_external_toc",
+    "sphinx_thebe",
     "sphinxcontrib.autodoc_pydantic",
     "sphinxcontrib.redoc",
     "sphinx_tabs.tabs",
@@ -103,6 +104,12 @@
     "replacements",
 ]
 
+# Thebe configuration for launching notebook cells within the docs.
+thebe_config = {
+    "selector": "div.highlight",
+    "repository_url": "https://github.com/ray-project/ray",
+    "repository_branch": "master",
+}
 
 # Cache notebook outputs in _build/.jupyter_cache
 # To prevent notebook execution, set this to "off". To force re-execution, set this to "force".
@@ -259,6 +266,11 @@
     "path_to_docs": "doc/source",
     "home_page_in_toc": False,
     "show_navbar_depth": 1,
+    "launch_buttons": {
+        "notebook_interface": "jupyterlab",
+        "binderhub_url": "https://mybinder.org",
+        "colab_url": "https://colab.research.google.com",
+    },
     "announcement": "<div class='topnav'></div>",
 }
 

diff --git a/doc/source/index.md b/doc/source/index.md
@@ -27,6 +27,7 @@ Learn how to install Ray, compute an example with the Ray Core API, and use Ray'
 **User guides**
 ^^^^^^^^^^^
 
+
 Learn about the key concepts and features. Get in-depth information about Ray's libraries and tooling.
 +++