[Datasets] Re-enable RayDP tests since arrow version limits removed (#…

…31124) Now that MLDataset has been made optional dependency, and PyArrow version limit has been removed, RayDP tests in ray can be re-enabled now. Signed-off-by: Zhi Lin <[email protected]>
ray-project · Feb 17, 2023 · c7a50b4 · c7a50b4
1 parent ed42885
commit c7a50b4
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 11 deletions.
diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml
@@ -302,6 +302,7 @@
   instance_size: medium
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/install-java.sh
     - DATA_PROCESSING_TESTING=1 ARROW_VERSION=nightly ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -312,6 +313,7 @@
   instance_size: medium
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/install-java.sh
     - DATA_PROCESSING_TESTING=1 ARROW_VERSION=10.* ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -322,6 +324,7 @@
   instance_size: medium
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/install-java.sh
     - DATA_PROCESSING_TESTING=1 ARROW_VERSION=9.* ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -332,6 +335,7 @@
   instance_size: medium
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/install-java.sh
     - DATA_PROCESSING_TESTING=1 ARROW_VERSION=8.* ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -342,6 +346,7 @@
   instance_size: medium
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/install-java.sh
     - DATA_PROCESSING_TESTING=1 ARROW_VERSION=7.* ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...
@@ -352,6 +357,7 @@
   instance_size: medium
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/install-java.sh
     - DATA_PROCESSING_TESTING=1 ARROW_VERSION=6.* ./ci/env/install-dependencies.sh
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/...

diff --git a/ci/env/install-java.sh b/ci/env/install-java.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Push caller's shell options (quietly)
+{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null
+
+set -euxo pipefail
+
+SCRIPT_DIR=$(builtin cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+WORKSPACE_DIR="${SCRIPT_DIR}/../.."
+
+sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk
+"${WORKSPACE_DIR}"/java/build-jar-multiplatform.sh linux
+
+# Pop caller's shell options (quietly)
+{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null
diff --git a/python/ray/data/tests/test_raydp_dataset.py b/python/ray/data/tests/test_raydp_dataset.py
@@ -1,8 +1,3 @@
-# Skipping entire test module:
-#  - Waiting for MLDataset to be made an optional dependency
-#  - See https://github.com/oap-project/raydp/pull/241
-
-"""
 import pytest
 import ray
 import raydp
@@ -12,7 +7,7 @@
 @pytest.fixture(scope="function")
 def spark(request):
     ray.init(num_cpus=2, include_dashboard=False)
-    spark_session = raydp.init_spark("test", 1, 1, "500 M")
+    spark_session = raydp.init_spark("test", 1, 1, "500M")
 
     def stop_all():
         raydp.stop_spark()
@@ -59,4 +54,3 @@ def test_raydp_to_torch_iter(spark):
     import sys
 
     sys.exit(pytest.main(["-v", __file__]))
-"""
diff --git a/python/requirements/data_processing/requirements.txt b/python/requirements/data_processing/requirements.txt
@@ -10,9 +10,6 @@ s3fs
 modin>=0.8.3;  python_version < '3.7'
 modin>=0.11.0; python_version >= '3.7'
 pytest-repeat
-# Uncomment when RayDP:
-# - Makes MLDataset an optional dependency.
-# - Removes pyarrow < 7.0.0 upperbound.
-# raydp-nightly==2022.6.30.dev1
+raydp>=0.0.dev0
 responses==0.13.4
 pymars>=0.8.3