Build from source in Jenkins (#8255)

ray-project · May 28, 2020 · b93d681 · b93d681
1 parent 448011f
commit b93d681
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 4 deletions.
diff --git a/ci/jenkins_tests/run_doc_tests.sh b/ci/jenkins_tests/run_doc_tests.sh
@@ -26,7 +26,11 @@ if [ "$DOCKER_SHA" == "" ]; then
     # Add Ray source
     git rev-parse HEAD > ./docker/tune_test/git-rev
     git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD)
-    DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test)
+    if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then
+      DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile)
+    else
+      DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test)
+    fi
 fi
 
 echo "Using Docker image" $DOCKER_SHA

diff --git a/ci/jenkins_tests/run_multi_node_tests.sh b/ci/jenkins_tests/run_multi_node_tests.sh
@@ -5,5 +5,6 @@ ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 export RUN_TUNE_TESTS=1
 export RUN_DOC_TESTS=1
 export RUN_SGD_TESTS=1
+export CI_BUILD_FROM_SOURCE=1
 
-bash $ROOT_DIR/entry_point.sh
+bash $ROOT_DIR/entry_point.sh
diff --git a/ci/jenkins_tests/run_sgd_tests.sh b/ci/jenkins_tests/run_sgd_tests.sh
@@ -26,7 +26,11 @@ if [ "$DOCKER_SHA" == "" ]; then
     # Add Ray source
     git rev-parse HEAD > ./docker/tune_test/git-rev
     git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD)
-    DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test)
+    if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then
+      DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile)
+    else
+      DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test)
+    fi
 fi
 
 echo "Using Docker image" $DOCKER_SHA

diff --git a/ci/jenkins_tests/run_tune_tests.sh b/ci/jenkins_tests/run_tune_tests.sh
@@ -26,7 +26,12 @@ if [ "$DOCKER_SHA" == "" ]; then
     # Add Ray source
     git rev-parse HEAD > ./docker/tune_test/git-rev
     git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD)
-    DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test)
+
+    if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then
+      DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile)
+    else
+      DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test)
+    fi
 fi
 
 echo "Using Docker image" $DOCKER_SHA

diff --git a/docker/tune_test/build_from_source.Dockerfile b/docker/tune_test/build_from_source.Dockerfile
@@ -0,0 +1,24 @@
+# The stress_test Docker image build a self-contained Ray instance for launching Ray.
+
+FROM ray-project/base-deps
+
+RUN pip install -U pip
+
+# We install this after the latest wheels -- this should not override the latest wheels.
+# Needed to run Tune example with a 'plot' call - which does not actually render a plot, but throws an error.
+RUN pip install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
+RUN pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+# We port the source code in so that we run the most up-to-date stress tests.
+ADD ray.tar /ray
+ADD git-rev /ray/git-rev
+
+RUN bash /ray/ci/travis/install-bazel.sh
+RUN echo 'build --remote_cache="https://storage.googleapis.com/ray-bazel-cache"' >> $HOME/.bashrc
+RUN echo 'build --remote_upload_local_results=false' >> $HOME/.bashrc
+RUN cd /ray/python; pip install -e . --verbose
+
+WORKDIR /ray