Kiko/annotate ray

python/build-wheel-macos.sh

@@ -14,24 +14,36 @@ MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
 DOWNLOAD_DIR=python_downloads
 
 NODE_VERSION="14"
+# BONSAI changes begin - python version 3.6.1 removed
 PY_VERSIONS=("3.7.0"
              "3.8.2")
+# BONSAI changes end
+# BONSAI changes begin - "python-3.6.1-macosx10.6.pkg" removed
 PY_INSTS=("python-3.7.0-macosx10.6.pkg"
           "python-3.8.2-macosx10.9.pkg")
+# BONSAI changes end
+# BONSAI changes begin - Python 3.6 removed
 PY_MMS=("3.7"
         "3.8")
+# BONSAI changes end
 
 # The minimum supported numpy version is 1.14, see
 # https://issues.apache.org/jira/browse/ARROW-3141
+# BONSAI changes begin
+# NUMPY_VERSIONS=("1.14.5"
+#                 "1.14.5"
+#                 "1.14.5")
 NUMPY_VERSIONS=("1.19.2"
                 "1.19.2")
+# BONSAI changes end
+
 
 ./ci/travis/install-bazel.sh
 
 mkdir -p $DOWNLOAD_DIR
 mkdir -p .whl
-
-# In azure pipelines or github acions, we don't need to install node
+# BONSAI changes begin - added check if Node already installed and don't install in that case
+# In azure pipelines or github actions, we don't need to install node
 if [ -x "$(command -v npm)" ]; then
   echo "Node already installed"
   npm -v
@@ -41,6 +53,7 @@ else
   nvm install $NODE_VERSION
   nvm use node
 fi
+# BONSAI changes end
 
 # Build the dashboard so its static assets can be included in the wheel.
 # TODO(mfitton): switch this back when deleting old dashboard code.
@@ -58,7 +71,9 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
   # The -f flag is passed twice to also run git clean in the arrow subdirectory.
   # The -d flag removes directories. The -x flag ignores the .gitignore file,
   # and the -e flag ensures that we don't remove the .whl directory.
+  # BONSAI changes begin - added code
   git config --global --add safe.directory /ray
+  # BONSAI changes end
   git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR -e python/ray/new_dashboard/client -e dashboard/client
 
   # Install Python.
@@ -76,7 +91,9 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
 
   pushd python
     # Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade.
-    $PIP_CMD install --upgrade setuptools
+    # BONSAI changes begin - pinned setuptools to 41.0.0
+    $PIP_CMD install setuptools==41.0.0
+    # BONSAI changes end
     # Install setuptools_scm because otherwise when building the wheel for
     # Python 3.6, we see an error.
     $PIP_CMD install -q setuptools_scm==3.1.0
@@ -94,7 +111,9 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
     fi
     # Add the correct Python to the path and build the wheel. This is only
     # needed so that the installation finds the cython executable.
+    # BONSAI changes begin -     PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel
     MACOSX_DEPLOYMENT_TARGET=10.15 PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $PYTHON_EXE setup.py bdist_wheel
+    # BONSAI changes end
     mv dist/*.whl ../.whl/
   popd
 done

python/build-wheel-manylinux2014.sh

@@ -12,20 +12,31 @@ EOF
 chmod +x /usr/bin/nproc
 
 NODE_VERSION="14"
+# BONSAI changes begin - removed "cp36-cp36m"
 PYTHONS=("cp37-cp37m"
          "cp38-cp38")
+# BONSAI changes end
+
 
 # The minimum supported numpy version is 1.14, see
 # https://issues.apache.org/jira/browse/ARROW-3141
+# BONSAI changes begin
+# NUMPY_VERSIONS=("1.14.5"
+#                 "1.14.5"
+#                 "1.14.5")
 NUMPY_VERSIONS=("1.19.2"
                 "1.19.2")
-
+# BONSAI changes end
+# BONSAI changes begin - added code
 yum -y update
+# BONSAI changes end
 yum -y install unzip zip sudo
 yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel xz
 yum -y install openssl
+# BONSAI changes begin - added code
 yum -y install curl
 yum install -y gcc-c++ make
+# BONSAI changes end
 
 java -version
 java_bin=$(readlink -f "$(command -v java)")
@@ -47,6 +58,7 @@ if [[ -n "${RAY_INSTALL_JAVA:-}" ]]; then
   unset RAY_INSTALL_JAVA
 fi
 
+# BONSAI changes begin
 # In azure pipelines or github acions, we don't need to install node
 if [ -x "$(command -v npm)" ]; then
   echo "Node already installed"
@@ -74,16 +86,19 @@ else
   # node –v
   npm -v
 fi
+# BONSAI changes end
 
 # Build the dashboard so its static assets can be included in the wheel.
 # TODO(mfitton): switch this back when deleting old dashboard code.
 pushd python/ray/new_dashboard/client
+  # BONSAI changes begin
   pwd
   echo "Run npm ci"
   npm ci
   echo "Run npm run build"
   npm run build
   echo "Done running npm run build"
+  # BONSAI changes end
 popd
 set -x
 
@@ -96,7 +111,9 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do
   # The -d flag removes directories. The -x flag ignores the .gitignore file,
   # and the -e flag ensures that we don't remove the .whl directory, the
   # dashboard directory and jars directory.
+  # BONSAI changes begin - added code
   git config --global --add safe.directory /ray
+  # BONSAI changes end
   git clean -f -f -x -d -e .whl -e python/ray/new_dashboard/client -e dashboard/client -e python/ray/jars
 
   pushd python
@@ -110,9 +127,12 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do
       echo "TRAVIS_COMMIT variable not set - required to populated ray.__commit__."
       exit 1
     fi
-
+    # BONSAI changes begin
+    # PATH=/opt/python/${PYTHON}/bin:/root/bazel-3.2.0/output:$PATH \
+    #    /opt/python/"${PYTHON}"/bin/python setup.py bdist_wheel
     PATH=/opt/python/${PYTHON}/bin:/root/bazel-3.4.1/output:$PATH \
     /opt/python/"${PYTHON}"/bin/python setup.py bdist_wheel
+    # BONSAI changes end
     # In the future, run auditwheel here.
     mv dist/*.whl ../.whl/
   popd
@@ -127,5 +147,7 @@ for path in .whl/*.whl; do
 done
 
 # Clean the build output so later operations is on a clean directory.
+# BONSAI changes begin - added code
 git config --global --add safe.directory /ray
+# BONSAI changes end
 git clean -f -f -x -d -e .whl -e python/ray/dashboard/client

python/ray/_private/memory_monitor.py

@@ -38,6 +38,7 @@ def get_message(used_gb, total_gb, threshold):
         pids = psutil.pids()
         proc_stats = []
         for pid in pids:
+            # BONSAI changes begin
             try:
                 proc = psutil.Process(pid)
                 proc_stats.append((get_rss(proc.memory_info()), pid,
@@ -47,6 +48,7 @@ def get_message(used_gb, total_gb, threshold):
                 # respect 100 character limit
                 msg = msg[:100]
                 proc_stats.append(0, pid, msg)
+            # BONSAI changes end
         proc_str = "PID\tMEM\tCOMMAND"
         for rss, pid, cmdline in sorted(proc_stats, reverse=True)[:10]:
             proc_str += "\n{}\t{}GiB\t{}".format(

python/ray/test_utils.py

@@ -193,8 +193,12 @@ def run_string_as_driver(driver_script):
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT)
     with proc:
+        # BONSAI changes begin
+        # output = proc.communicate(driver_script.encode("ascii"))[0]
+        #   if proc.returncode:
         output, error = proc.communicate(driver_script.encode("ascii"))
         if proc.returncode or error is not None:
+        # BONSAI changes end
             print(ray._private.utils.decode(output))
             raise subprocess.CalledProcessError(proc.returncode, proc.args,
                                                 output, proc.stderr)

python/ray/tests/BUILD

@@ -5,9 +5,11 @@ SRCS = [] + select({
         # TODO(mehrdadn): This should be added for all platforms once resulting errors are fixed
         "**/conftest.py",
     ]),
+    # BONSAI changes begin - "//conditions:default": [],
     "//conditions:default": glob([
         "**/conftest.py",
     ]),
+    # BONSAI changes end
 })
 
 py_test_module_list(
@@ -40,10 +42,14 @@ py_test_module_list(
     "test_global_gc.py",
     "test_mldataset.py",
   ],
+  # BONSAI changes begin - size = "medium",
   size = "large",
+  # BONSAI changes end
   extra_srcs = SRCS,
   tags = ["exclusive", "medium_size_python_tests_a_to_j"],
+  # BONSAI changes begin - added code
   data = glob(["mnist_784_100_samples.pkl"]),
+  # BONSAI changes end
   deps = ["//:ray_lib"],
 )
 
@@ -114,6 +120,7 @@ py_test_module_list(
   extra_srcs = SRCS,
   tags = ["exclusive"],
   deps = ["//:ray_lib"],
+  # BONSAI changes begin - added code
   data = [
     "additional_property.yaml",
     "//:python/ray/autoscaler/aws/example-full.yaml",
@@ -125,6 +132,7 @@ py_test_module_list(
     "//:python/ray/autoscaler/aws/example-minimal.yaml",
     "//:python/ray/autoscaler/aws/example-multi-node-type.yaml",
   ],
+  # BONSAI changes end
 )
 
 py_test_module_list(
@@ -139,11 +147,12 @@ py_test_module_list(
   deps = ["//:ray_lib"],
   data = glob(["test_cli_patterns/*.*"], allow_empty=False),
 )
-
+# BONSAI changes begin - added code
 TEST_RUNTIME_ENV_SOURCES = [
     "//:python/ray/experimental/packaging/example_pkg/my_pkg/stubs.py",
     "//:python/ray/experimental/packaging/example_pkg/my_pkg/impl/__init__.py",
 ]
+# BONSAI changes end
 
 py_test_module_list(
   files = [
@@ -152,9 +161,13 @@ py_test_module_list(
     "test_runtime_env.py",
   ],
   size = "large",
+  # BONSAI changes begin -   extra_srcs = SRCS,
   extra_srcs = SRCS + TEST_RUNTIME_ENV_SOURCES,
+  # BONSAI changes end
   deps = ["//:ray_lib"],
+  # BONSAI changes begin - added code
   data = ["//:python/ray/experimental/packaging/example_pkg/ray_pkg.yaml"],
+  # BONSAI changes end
 )
 
 # TODO(barakmich): aws/ might want its own buildfile, or

python/ray/tests/test_autoscaler_yaml.py

@@ -178,8 +178,12 @@ def testValidateDefaultConfigAWSMultiNodeTypes(self):
                     "Config did not pass multi node types auto fill test!")
 
     def testValidateNetworkConfig(self):
+        # BONSAI changes begin
+        # web_yaml = "https://raw.githubusercontent.com/ray-project/ray/" \
+        #   "master/python/ray/autoscaler/aws/example-full.yaml"
         web_yaml = "https://raw.githubusercontent.com/BonsaiAI/ray/" \
             "master/python/ray/autoscaler/aws/example-full.yaml"
+        # BONSAI changes end
         response = urllib.request.urlopen(web_yaml, timeout=5)
         content = response.read()
         with tempfile.TemporaryFile() as f:

python/ray/tests/test_basic.py

@@ -6,7 +6,9 @@
 import time
 
 import numpy as np
+# BONSAI changes begin - added import
 import platform
+# BONSAI changes end
 import pytest
 
 import ray.cluster_utils
@@ -251,9 +253,11 @@ def foo():
             }).remote())
 
     to_check = ["CPU", "GPU", "memory", "custom1"]
+    # BONSAI changes begin - added if clause
     if "linux" in platform.platform().lower():
         # TODO(Edi): we have to fix the agents used by the pipelines to allow this
         #            kind of tests in mac.
+    # BONSAI changes end
         for key in to_check:
             assert without_options[key] != with_options[key], key
         assert without_options != with_options

python/ray/tests/test_client.py

@@ -338,6 +338,7 @@ def print_on_stderr_and_stdout(s):
         time.sleep(1)
         print_on_stderr_and_stdout.remote("Hello world")
         time.sleep(1)
+        # BONSAI changes begin
         # NOTE (Ruofan): The test is flakey and Ray fixes this test in Ray 1+:
         # https://github.com/ray-project/ray/pull/19232/files
         # We apply the fix here. Later once we updated the Ray version, we SHOULD
@@ -347,6 +348,7 @@ def print_on_stderr_and_stdout(s):
             if "Hello world" in msg:
                 num_hello += 1
         assert num_hello == 2, f"Invalid logs: {log_msgs}"
+        # BONSAI changes end
 
 
 @pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")

python/ray/tests/test_debug_tools.py

@@ -25,8 +25,10 @@ def ray_gdb_start():
 
 @pytest.mark.skipif(
     sys.platform != "linux" and sys.platform != "linux2",
+    # BONSAI changes begin - added decorator
     reason="TODO(Edi): This test requires Linux.")
 # @pytest.mark.skip(reason="Too flaky.")
+# BONSAI changes end
 def test_raylet_gdb(ray_gdb_start):
     # ray_gdb_start yields the expected process name
     ray.init(num_cpus=1)