Merge branch 'master' into gcs-gke-bucket-doc

.buildkite/lint.rayci.yml

@@ -67,3 +67,8 @@ steps:
       - RAY_DISABLE_EXTRA_CPP=1 pip install -e python/[all]
       - ./ci/lint/check_api_annotations.py
     depends_on: forge
+
+  - label: ":lint-roller: lint: documentation style"
+    commands:
+      - ./ci/lint/check-documentation-style.sh
+    depends_on: forge

.buildkite/pipeline.build.yml

@@ -334,6 +334,58 @@
     - echo "--- Running the script with fault of networking delay"
     - ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/runtime_env.yaml --working-dir python/ray/tests/chaos -- python potato_passer.py --num-actors=3 --pass-times=1000 --sleep-secs=0.01
 
+
+- label: ":kubernetes: :mending_heart: :runner: chaos network delay many job submissions"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: medium
+  commands:
+    - |
+      cleanup() {
+        if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi
+        kind delete cluster
+      }
+      trap cleanup EXIT
+    - ./ci/env/install-minimal.sh 3.8
+    - PYTHON=3.8 ./ci/env/install-dependencies.sh
+    # Specifying above somehow messes up the Ray install.
+    # Uninstall and re-install Ray so that we can use Ray Client.
+    # (Remove thirdparty_files to sidestep an issue with psutil.)
+    - pip uninstall -y ray && rm -rf /ray/python/ray/thirdparty_files
+    - pip install -e /ray/python
+    - echo "--- Setting up local kind cluster."
+    - ./ci/k8s/prep-k8s-environment.sh
+    - ./ci/k8s/prep-helm.sh
+    - echo "--- Building py38-cpu Ray image for the test."
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker
+    - python ci/build/build-docker-images.py --py-versions py38 --device-types cpu --build-type LOCAL --build-base
+    # Tag the image built in the last step. We want to be sure to distinguish the image from the real Ray nightly.
+    - docker tag rayproject/ray:nightly-py38-cpu ray-ci:kuberay-test
+    # Load the image into the kind node
+    - kind load docker-image ray-ci:kuberay-test
+    # Helm install KubeRay
+    - echo "--- Installing KubeRay operator and cluser."
+    - helm repo add kuberay https://ray-project.github.io/kuberay-helm/
+    - helm install kuberay-operator kuberay/kuberay-operator
+    - kubectl wait pod  -l app.kubernetes.io/name=kuberay-operator --for=condition=Ready=True  --timeout=5m
+    # We are in m4i.xlarge and have 4 cpus. Can't have too many nodes.
+    - helm install raycluster kuberay/ray-cluster --set image.repository=ray-ci --set image.tag=kuberay-test --set worker.replicas=2 --set worker.resources.limits.cpu=500m --set worker.resources.requests.cpu=500m --set head.resources.limits.cpu=500m --set head.resources.requests.cpu=500m
+    - kubectl wait pod -l ray.io/cluster=raycluster-kuberay --for=condition=Ready=True --timeout=5m
+    - kubectl port-forward --address 0.0.0.0 service/raycluster-kuberay-head-svc 8265:8265 &
+    # Helm install chaos-mesh
+    - echo "--- Installing chaos-mesh operator and CR."
+    - helm repo add chaos-mesh https://charts.chaos-mesh.org
+    - kubectl create ns chaos-mesh
+    - helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-mesh --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --version 2.6.1
+    - kubectl wait pod  --namespace chaos-mesh  -l app.kubernetes.io/instance=chaos-mesh --for=condition=Ready=True
+    - echo "--- Running the script without faults"
+    - ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/runtime_env.yaml --working-dir python/ray/tests/chaos -- python potato_passer.py --num-actors=3 --pass-times=3 --sleep-secs=0.01
+    # Now add the delay, rerun the job
+    - kubectl apply -f python/ray/tests/chaos/chaos_network_delay.yaml
+    - echo "--- Running the script 100 times with fault of networking delay"
+    - for i in {1..100}; do ray job submit --address http://localhost:8265 --runtime-env python/ray/tests/chaos/runtime_env.yaml --working-dir python/ray/tests/chaos -- python potato_passer.py --num-actors=3 --pass-times=3 --sleep-secs=0.01; done
+
+
 # TODO: write a test that needs bandwidth heavy lifting
 - label: ":kubernetes: :mending_heart: chaos bandwidth test"
   conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]

.buildkite/pipeline.ml.yml

@@ -8,7 +8,6 @@
     - DATA_PROCESSING_TESTING=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu,-hdfs
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=0
       python/ray/air/...
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=ray_air 
       python/ray/data/...
@@ -28,7 +27,6 @@
     - bazel test --config=ci $(./ci/run/bazel_export_options)
       --build_tests_only
       --test_tag_filters=team:ml
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=0
       release/...
 
 
@@ -281,8 +279,7 @@
     - ./ci/env/env_info.sh
     - ./ci/run/run_bazel_test_with_sharding.sh
       --config=ci $(./ci/run/bazel_export_options) --build_tests_only
-      --test_tag_filters=-medium_instance,-soft_imports,-gpu_only,-rllib,-multinode,-new_storage
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=0
+      --test_tag_filters=-medium_instance,-soft_imports,-gpu_only,-rllib,-multinode
       python/ray/tune/...
 
 - label: ":octopus: Tune tests and examples (medium)"
@@ -293,8 +290,7 @@
     - TUNE_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
-      --test_tag_filters=medium_instance,-soft_imports,-gpu_only,-rllib,-multinode,-new_storage
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=0
+      --test_tag_filters=medium_instance,-soft_imports,-gpu_only,-rllib,-multinode
       python/ray/tune/...
 
 - label: ":octopus: :spiral_note_pad: New output: Tune tests and examples (small)"
@@ -307,9 +303,8 @@
     - ./ci/env/env_info.sh
     - ./ci/run/run_bazel_test_with_sharding.sh
       --config=ci $(./ci/run/bazel_export_options) --build_tests_only
-      --test_tag_filters=-medium_instance,-soft_imports,-gpu_only,-rllib,-multinode,-new_storage
+      --test_tag_filters=-medium_instance,-soft_imports,-gpu_only,-rllib,-multinode
       --test_env=AIR_VERBOSITY=1
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=0
       python/ray/tune/...
 
 - label: ":octopus: :spiral_note_pad: New output: Tune tests and examples (medium)"
@@ -320,9 +315,8 @@
     - TUNE_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
-      --test_tag_filters=medium_instance,-soft_imports,-gpu_only,-rllib,-multinode,-new_storage
+      --test_tag_filters=medium_instance,-soft_imports,-gpu_only,-rllib,-multinode
       --test_env=AIR_VERBOSITY=1
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=0
       python/ray/tune/...
 
 
@@ -344,50 +338,11 @@
     - TUNE_TESTING=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only 
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=0
       python/ray/tests/xgboost/...
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only 
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=0
       python/ray/tests/horovod/...
 
 
-
-##### STORAGE REFACTOR
-
-
-- label: ":octopus: :floppy_disk: New persistence mode: Tune tests and examples (small)"
-  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"]
-  instance_size: small
-  parallelism: 3
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
-    - ./ci/env/env_info.sh
-    - ./ci/run/run_bazel_test_with_sharding.sh
-      --config=ci $(./ci/run/bazel_export_options) --build_tests_only
-      --test_tag_filters=-medium_instance,-soft_imports,-gpu_only,-rllib,-multinode,-no_new_storage
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=1
-      python/ray/tune/...
-
-- label: ":octopus: :floppy_disk: New persistence mode: Tune tests and examples (medium)"
-  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"]
-  instance_size: medium
-  commands:
-    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - TUNE_TESTING=1 DATA_PROCESSING_TESTING=1 ./ci/env/install-dependencies.sh
-    - ./ci/env/env_info.sh
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
-      --test_tag_filters=medium_instance,-soft_imports,-gpu_only,-rllib,-multinode,-no_new_storage
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=1
-      python/ray/tune/...
-
-
-###### END STORAGE REFACTOR
-
-
-
-
-
 # TODO(amogkam): Re-enable Ludwig tests after Ludwig supports Ray 2.0
 #- label: ":octopus: Ludwig tests and examples. Python 3.7"
 #  conditions: ["NO_WHEELS_REQUIRED", "RAY_CI_TUNE_AFFECTED"]

.vale.ini

@@ -1,4 +1,4 @@
-StylesPath = .github/styles
+StylesPath = .vale/styles
 
 Vocab = General, Data
 

.github/styles/Google/Acronyms.yml → .vale/styles/Google/Acronyms.yml

@@ -1,7 +1,7 @@
 extends: conditional
 message: "Spell out '%s', if it's unfamiliar to the audience."
 link: 'https://developers.google.com/style/abbreviations'
-level: warning
+level: error
 ignorecase: false
 # Ensures that the existence of 'first' implies the existence of 'second'.
 first: '\b([A-Z]{3,5})\b'

.github/styles/Google/Contractions.yml → .vale/styles/Google/Contractions.yml

@@ -1,7 +1,7 @@
 extends: substitution
 message: "Use '%s' instead of '%s'."
 link: 'https://developers.google.com/style/contractions'
-level: warning
+level: error
 ignorecase: true
 action:
   name: replace

.github/styles/Google/Ellipses.yml → .vale/styles/Google/Ellipses.yml

@@ -2,7 +2,7 @@ extends: existence
 message: "In general, don't use an ellipsis."
 link: 'https://developers.google.com/style/ellipses'
 nonword: true
-level: warning
+level: error
 action:
   name: remove
 tokens:

.github/styles/Google/FirstPerson.yml → .vale/styles/Google/FirstPerson.yml

@@ -2,7 +2,7 @@ extends: existence
 message: "Avoid first-person pronouns such as '%s'."
 link: 'https://developers.google.com/style/pronouns#personal-pronouns'
 ignorecase: true
-level: warning
+level: error
 nonword: true
 tokens:
   - (?:^|\s)I\s

.github/styles/Google/HeadingPunctuation.yml → .vale/styles/Google/HeadingPunctuation.yml

@@ -2,7 +2,7 @@ extends: existence
 message: "Don't put a period at the end of a heading."
 link: 'https://developers.google.com/style/capitalization#capitalization-in-titles-and-headings'
 nonword: true
-level: warning
+level: error
 scope: heading
 action:
   name: edit

.github/styles/Google/Headings.yml → .vale/styles/Google/Headings.yml

@@ -1,7 +1,7 @@
 extends: capitalization
 message: "'%s' should use sentence-style capitalization."
 link: 'https://developers.google.com/style/capitalization#capitalization-in-titles-and-headings'
-level: warning
+level: error
 scope: heading
 match: $sentence
 indicators:

.github/styles/Google/OxfordComma.yml → .vale/styles/Google/OxfordComma.yml

@@ -2,6 +2,6 @@ extends: existence
 message: "Use the Oxford comma in '%s'."
 link: 'https://developers.google.com/style/commas'
 scope: sentence
-level: warning
+level: error
 tokens:
   - '(?:[^,]+,){1,}\s\w+\s(?:and|or)'

.github/styles/Google/Ranges.yml → .vale/styles/Google/Ranges.yml

@@ -2,6 +2,6 @@ extends: existence
 message: "Don't add words such as 'from' or 'between' to describe a range of numbers."
 link: 'https://developers.google.com/style/hyphens'
 nonword: true
-level: warning
+level: error
 tokens:
   - '(?:from|between)\s\d+\s?-\s?\d+'

.github/styles/Google/Spacing.yml → .vale/styles/Google/Spacing.yml

@@ -1,7 +1,7 @@
 extends: existence
 message: "'%s' should have one space."
 link: 'https://developers.google.com/style/sentence-spacing'
-level: warning
+level: error
 nonword: true
 action:
   name: remove

.github/styles/Google/Spelling.yml → .vale/styles/Google/Spelling.yml

@@ -2,7 +2,7 @@ extends: existence
 message: "In general, use American spelling instead of '%s'."
 link: 'https://developers.google.com/style/spelling'
 ignorecase: true
-level: warning
+level: error
 tokens:
   - '(?:\w+)nised?'
   - 'colour'

.github/styles/Google/We.yml → .vale/styles/Google/We.yml

@@ -1,7 +1,7 @@
 extends: existence
 message: "Try to avoid using first-person plural like '%s'."
 link: 'https://developers.google.com/style/pronouns#personal-pronouns'
-level: warning
+level: error
 ignorecase: true
 tokens:
   - we

.github/styles/Google/Will.yml → .vale/styles/Google/Will.yml

@@ -2,7 +2,7 @@ extends: existence
 message: "Avoid using '%s'."
 link: 'https://developers.google.com/style/tense'
 ignorecase: true
-level: warning
+level: error
 tokens:
   - will
   - "'ll"

.github/styles/Google/WordList.yml → .vale/styles/Google/WordList.yml

@@ -1,7 +1,7 @@
 extends: substitution
 message: "Use '%s' instead of '%s'."
 link: "https://developers.google.com/style/word-list"
-level: warning
+level: error
 ignorecase: false
 action:
   name: replace

.github/styles/Vocab/Data/accept.txt → .vale/styles/Vocab/Data/accept.txt

@@ -6,6 +6,7 @@ Spotify('s)?
 TFRecord(s)?
 UDF(s)?
 [Dd]atasource
+[Dd]iscretizer(s)?
 [Gg]roupby
 [Ii]ndexable
 [Ii]ngest

ci/lint/check-documentation-style.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+VALE_BIN=$(mktemp -d)
+wget https://github.com/errata-ai/vale/releases/download/v2.28.0/vale_2.28.0_Linux_64-bit.tar.gz -P "$VALE_BIN"
+tar -xvzf "$VALE_BIN"/vale_2.28.0_Linux_64-bit.tar.gz -C "$VALE_BIN" vale
+"$VALE_BIN"/vale doc/source/data
+rm -rf "$VALE_BIN"