From 76ae690b44b136c741d299bc059c3dbbefe38f88 Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Thu, 15 Dec 2022 08:49:45 +0000 Subject: [PATCH 01/10] uncomment raydp test Signed-off-by: Zhi Lin --- python/ray/data/tests/test_raydp_dataset.py | 6 ------ python/requirements/data_processing/requirements.txt | 5 +---- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/python/ray/data/tests/test_raydp_dataset.py b/python/ray/data/tests/test_raydp_dataset.py index c299f13c951f..694a46065ed7 100644 --- a/python/ray/data/tests/test_raydp_dataset.py +++ b/python/ray/data/tests/test_raydp_dataset.py @@ -1,8 +1,3 @@ -# Skipping entire test module: -# - Waiting for MLDataset to be made an optional dependency -# - See https://github.com/oap-project/raydp/pull/241 - -""" import pytest import ray import raydp @@ -59,4 +54,3 @@ def test_raydp_to_torch_iter(spark): import sys sys.exit(pytest.main(["-v", __file__])) -""" diff --git a/python/requirements/data_processing/requirements.txt b/python/requirements/data_processing/requirements.txt index d6ca94d9c68f..0b9ad01c56b4 100644 --- a/python/requirements/data_processing/requirements.txt +++ b/python/requirements/data_processing/requirements.txt @@ -11,9 +11,6 @@ s3fs modin>=0.8.3; python_version < '3.7' modin>=0.11.0; python_version >= '3.7' pytest-repeat -# Uncomment when RayDP: -# - Makes MLDataset an optional dependency. -# - Removes pyarrow < 7.0.0 upperbound. -# raydp-nightly==2022.6.30.dev1 +raydp-nightly responses==0.13.4 pymars>=0.8.3 From a8b18acc3236971549699b07121c355b70943480 Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Tue, 3 Jan 2023 14:33:49 +0000 Subject: [PATCH 02/10] install java in ml docker image Signed-off-by: Zhi Lin --- ci/docker/base.ml.Dockerfile | 5 +++++ ci/docker/ml.Dockerfile | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/docker/base.ml.Dockerfile b/ci/docker/base.ml.Dockerfile index 405ec1df0ca4..55e33943474b 100644 --- a/ci/docker/base.ml.Dockerfile +++ b/ci/docker/base.ml.Dockerfile @@ -1,6 +1,11 @@ ARG DOCKER_IMAGE_BASE_TEST FROM $DOCKER_IMAGE_BASE_TEST +ENV RAY_INSTALL_JAVA=1 + +RUN apt-get install -y -qq \ + maven openjdk-8-jre openjdk-8-jdk + # Move out of working dir /ray # Delete stale data WORKDIR / diff --git a/ci/docker/ml.Dockerfile b/ci/docker/ml.Dockerfile index fcdf7aeb9d14..10a91e6d7e68 100644 --- a/ci/docker/ml.Dockerfile +++ b/ci/docker/ml.Dockerfile @@ -13,6 +13,6 @@ WORKDIR /ray COPY . . # Install Ray -RUN SKIP_BAZEL_BUILD=1 RAY_INSTALL_JAVA=0 bash --login -i -c -- "python3 -m pip install -e /ray/python/" +RUN SKIP_BAZEL_BUILD=1 RAY_INSTALL_JAVA=1 bash --login -i -c -- "python3 -m pip install -e /ray/python/" RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh From 47b625565fe28dd7315dbb810b8204b04f3e7865 Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Sun, 29 Jan 2023 12:34:37 +0000 Subject: [PATCH 03/10] install java in script Signed-off-by: Zhi Lin --- ci/env/install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh index 834781993f96..0cf867d8c079 100755 --- a/ci/env/install-dependencies.sh +++ b/ci/env/install-dependencies.sh @@ -54,7 +54,7 @@ install_base() { curl -f -s -L -R https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - || true sudo apt-get update -qq pkg_install_helper build-essential curl unzip libunwind-dev python3-pip python3-setuptools \ - tmux gdb + tmux gdb maven openjdk-8-jre openjdk-8-jdk if [ "${LINUX_WHEELS-}" = 1 ]; then pkg_install_helper docker if [ -n "${TRAVIS-}" ]; then From ce2589026008c3bcd681d42f4d543c05940c65d6 Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Mon, 13 Feb 2023 14:14:41 +0000 Subject: [PATCH 04/10] install raydp nightly nightly has been merged into raydp Signed-off-by: Zhi Lin --- python/requirements/data_processing/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements/data_processing/requirements.txt b/python/requirements/data_processing/requirements.txt index 0b9ad01c56b4..cfe3344cd8c8 100644 --- a/python/requirements/data_processing/requirements.txt +++ b/python/requirements/data_processing/requirements.txt @@ -11,6 +11,6 @@ s3fs modin>=0.8.3; python_version < '3.7' modin>=0.11.0; python_version >= '3.7' pytest-repeat -raydp-nightly +raydp>=0.0.dev0 responses==0.13.4 pymars>=0.8.3 From b1e0685d010e3426c13b9dd0277810f7a1b73bce Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Thu, 16 Feb 2023 16:02:47 +0000 Subject: [PATCH 05/10] address comments Signed-off-by: Zhi Lin --- .buildkite/pipeline.ml.yml | 5 +++++ ci/docker/base.ml.Dockerfile | 5 ----- ci/docker/ml.Dockerfile | 2 +- ci/env/install-dependencies.sh | 2 +- ci/env/install-java.sh | 12 ++++++++++++ 5 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 ci/env/install-java.sh diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml index 9ba30cd73606..78ead8bfc6ea 100644 --- a/.buildkite/pipeline.ml.yml +++ b/.buildkite/pipeline.ml.yml @@ -298,6 +298,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=10.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -308,6 +309,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=9.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -318,6 +320,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=8.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -328,6 +331,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=7.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -338,6 +342,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=6.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... diff --git a/ci/docker/base.ml.Dockerfile b/ci/docker/base.ml.Dockerfile index 55e33943474b..405ec1df0ca4 100644 --- a/ci/docker/base.ml.Dockerfile +++ b/ci/docker/base.ml.Dockerfile @@ -1,11 +1,6 @@ ARG DOCKER_IMAGE_BASE_TEST FROM $DOCKER_IMAGE_BASE_TEST -ENV RAY_INSTALL_JAVA=1 - -RUN apt-get install -y -qq \ - maven openjdk-8-jre openjdk-8-jdk - # Move out of working dir /ray # Delete stale data WORKDIR / diff --git a/ci/docker/ml.Dockerfile b/ci/docker/ml.Dockerfile index 10a91e6d7e68..3e8258727b48 100644 --- a/ci/docker/ml.Dockerfile +++ b/ci/docker/ml.Dockerfile @@ -13,6 +13,6 @@ WORKDIR /ray COPY . . # Install Ray -RUN SKIP_BAZEL_BUILD=1 RAY_INSTALL_JAVA=1 bash --login -i -c -- "python3 -m pip install -e /ray/python/" +RUN SKIP_BAZEL_BUILD=1 bash --login -i -c -- "python3 -m pip install -e /ray/python/" RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh index 0cf867d8c079..834781993f96 100755 --- a/ci/env/install-dependencies.sh +++ b/ci/env/install-dependencies.sh @@ -54,7 +54,7 @@ install_base() { curl -f -s -L -R https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - || true sudo apt-get update -qq pkg_install_helper build-essential curl unzip libunwind-dev python3-pip python3-setuptools \ - tmux gdb maven openjdk-8-jre openjdk-8-jdk + tmux gdb if [ "${LINUX_WHEELS-}" = 1 ]; then pkg_install_helper docker if [ -n "${TRAVIS-}" ]; then diff --git a/ci/env/install-java.sh b/ci/env/install-java.sh new file mode 100644 index 000000000000..432794c4e77c --- /dev/null +++ b/ci/env/install-java.sh @@ -0,0 +1,12 @@ +sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk +#!/usr/bin/env bash + +# Push caller's shell options (quietly) +{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null + +set -euxo pipefail + +SCRIPT_DIR=$(builtin cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +WORKSPACE_DIR="${SCRIPT_DIR}/../.." + +"${WORKSPACE_DIR}"/java/build-jar-multiplatform.sh linux \ No newline at end of file From 4ba2d2abb8a1c7000c799d71db42ae48b17489bd Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Thu, 16 Feb 2023 16:45:40 +0000 Subject: [PATCH 06/10] test build jar Signed-off-by: Zhi Lin --- .buildkite/pipeline.ml.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml index 78ead8bfc6ea..abe46a819421 100644 --- a/.buildkite/pipeline.ml.yml +++ b/.buildkite/pipeline.ml.yml @@ -288,6 +288,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=nightly ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -298,7 +299,8 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - ./ci/env/install-java.sh + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=10.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -309,7 +311,8 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - ./ci/env/install-java.sh + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=9.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -320,7 +323,8 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - ./ci/env/install-java.sh + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=8.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -331,7 +335,8 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - ./ci/env/install-java.sh + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=7.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -342,7 +347,8 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - ./ci/env/install-java.sh + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=6.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... From 79cb22b704204341cb0c99b341c3b5447b5cb5f9 Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Thu, 16 Feb 2023 19:44:11 +0000 Subject: [PATCH 07/10] fix test Signed-off-by: Zhi Lin --- python/ray/data/tests/test_raydp_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/data/tests/test_raydp_dataset.py b/python/ray/data/tests/test_raydp_dataset.py index 694a46065ed7..b759fc65ee14 100644 --- a/python/ray/data/tests/test_raydp_dataset.py +++ b/python/ray/data/tests/test_raydp_dataset.py @@ -7,7 +7,7 @@ @pytest.fixture(scope="function") def spark(request): ray.init(num_cpus=2, include_dashboard=False) - spark_session = raydp.init_spark("test", 1, 1, "500 M") + spark_session = raydp.init_spark("test", 1, 1, "500M") def stop_all(): raydp.stop_spark() From 322eb0dc2690ce45570cf0ae8bc9a42b41cac252 Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Fri, 17 Feb 2023 08:42:08 +0000 Subject: [PATCH 08/10] fix Signed-off-by: Zhi Lin --- .buildkite/pipeline.ml.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml index abe46a819421..330d03f47cee 100644 --- a/.buildkite/pipeline.ml.yml +++ b/.buildkite/pipeline.ml.yml @@ -288,6 +288,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=nightly ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh @@ -299,7 +300,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=10.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh @@ -311,7 +312,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=9.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh @@ -323,7 +324,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=8.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh @@ -335,7 +336,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=7.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh @@ -347,7 +348,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk + - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - ./java/build-jar-multiplatform.sh linux - DATA_PROCESSING_TESTING=1 ARROW_VERSION=6.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh From e28cfd61a6cef96f9d5971c77f4c711967090359 Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Fri, 17 Feb 2023 10:06:30 +0000 Subject: [PATCH 09/10] use install-java.sh Signed-off-by: Zhi Lin --- .buildkite/pipeline.ml.yml | 18 ++++++------------ ci/env/install-java.sh | 7 +++++-- 2 files changed, 11 insertions(+), 14 deletions(-) mode change 100644 => 100755 ci/env/install-java.sh diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml index 330d03f47cee..eee9b381fa99 100644 --- a/.buildkite/pipeline.ml.yml +++ b/.buildkite/pipeline.ml.yml @@ -288,8 +288,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - - ./java/build-jar-multiplatform.sh linux + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=nightly ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -300,8 +299,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - - ./java/build-jar-multiplatform.sh linux + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=10.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -312,8 +310,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - - ./java/build-jar-multiplatform.sh linux + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=9.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -324,8 +321,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - - ./java/build-jar-multiplatform.sh linux + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=8.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -336,8 +332,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - - ./java/build-jar-multiplatform.sh linux + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=7.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... @@ -348,8 +343,7 @@ instance_size: medium commands: - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT - - sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk - - ./java/build-jar-multiplatform.sh linux + - ./ci/env/install-java.sh - DATA_PROCESSING_TESTING=1 ARROW_VERSION=6.* ./ci/env/install-dependencies.sh - ./ci/env/env_info.sh - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-dataset_integration python/ray/data/... diff --git a/ci/env/install-java.sh b/ci/env/install-java.sh old mode 100644 new mode 100755 index 432794c4e77c..20d6c6e97b94 --- a/ci/env/install-java.sh +++ b/ci/env/install-java.sh @@ -1,4 +1,3 @@ -sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk #!/usr/bin/env bash # Push caller's shell options (quietly) @@ -9,4 +8,8 @@ set -euxo pipefail SCRIPT_DIR=$(builtin cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WORKSPACE_DIR="${SCRIPT_DIR}/../.." -"${WORKSPACE_DIR}"/java/build-jar-multiplatform.sh linux \ No newline at end of file +sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk +"${WORKSPACE_DIR}"/java/build-jar-multiplatform.sh linux + +# Pop caller's shell options (quietly) +{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null \ No newline at end of file From 31ace16209abb9782301e841c67278de8bf208ff Mon Sep 17 00:00:00 2001 From: Zhi Lin Date: Fri, 17 Feb 2023 12:44:14 +0000 Subject: [PATCH 10/10] revert change Signed-off-by: Zhi Lin --- ci/docker/ml.Dockerfile | 2 +- ci/env/install-java.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/ml.Dockerfile b/ci/docker/ml.Dockerfile index 3e8258727b48..fcdf7aeb9d14 100644 --- a/ci/docker/ml.Dockerfile +++ b/ci/docker/ml.Dockerfile @@ -13,6 +13,6 @@ WORKDIR /ray COPY . . # Install Ray -RUN SKIP_BAZEL_BUILD=1 bash --login -i -c -- "python3 -m pip install -e /ray/python/" +RUN SKIP_BAZEL_BUILD=1 RAY_INSTALL_JAVA=0 bash --login -i -c -- "python3 -m pip install -e /ray/python/" RUN RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh diff --git a/ci/env/install-java.sh b/ci/env/install-java.sh index 20d6c6e97b94..dfff099930e9 100755 --- a/ci/env/install-java.sh +++ b/ci/env/install-java.sh @@ -12,4 +12,4 @@ sudo apt-get install -y maven openjdk-8-jre openjdk-8-jdk "${WORKSPACE_DIR}"/java/build-jar-multiplatform.sh linux # Pop caller's shell options (quietly) -{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null \ No newline at end of file +{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null