Ensure Jenkins CI jobs use up-to-date base Docker images (#2009)

* Ensure Jenkins CI jobs use up-to-date base Docker images In #1951 we ensured that the Github workflows would use up-to-date base Docker images by building all images in every run and leveraging Docker caching to reduce build times. With this new change, we do the same thing for Jenkins jobs. See #1540 * Increase integration tests timeout * Correct file permissions to enable Docker caching
antrea-io · Apr 2, 2021 · 6637cf0 · 6637cf0
1 parent 117cd94
commit 6637cf0
Show file tree

Hide file tree

Showing 7 changed files with 88 additions and 48 deletions.
diff --git a/Makefile b/Makefile
@@ -21,12 +21,6 @@ UNAME_S := $(shell uname -s)
 USERID  := $(shell id -u)
 GRPID   := $(shell id -g)
 
-# If NO_PULL is set, base Docker images will not be pulled.
-# If DOCKER_REGISTRY is set, we always set NO_PULL.
-ifneq ($(DOCKER_REGISTRY),)
-	NO_PULL := 1
-endif
-
 .PHONY: bin
 bin:
 	@mkdir -p $(BINDIR)

diff --git a/build/images/base/build.sh b/build/images/base/build.sh
@@ -84,9 +84,22 @@ CNI_BINARIES_VERSION=$(head -n 1 ../deps/cni-binaries-version)
 
 if $PULL; then
     docker pull $PLATFORM_ARG ubuntu:20.04
-    docker pull $PLATFORM_ARG antrea/openvswitch:$OVS_VERSION
-    docker pull $PLATFORM_ARG antrea/cni-binaries:$CNI_BINARIES_VERSION || true
-    docker pull $PLATFORM_ARG antrea/base-ubuntu:$OVS_VERSION || true
+    IMAGES_LIST=(
+        "antrea/openvswitch:$OVS_VERSION"
+        "antrea/cni-binaries:$CNI_BINARIES_VERSION"
+        "antrea/base-ubuntu:$OVS_VERSION"
+    )
+    for image in "${IMAGES_LIST[@]}"; do
+        if [[ ${DOCKER_REGISTRY} == "" ]]; then
+            docker pull $PLATFORM_ARG "${image}" || true
+        else
+            rc=0
+            docker pull "${DOCKER_REGISTRY}/${image}" || rc=$?
+            if [[ $rc -eq 0 ]]; then
+                docker tag "${DOCKER_REGISTRY}/${image}" "${image}"
+            fi
+        fi
+    done
 fi
 
 docker build $PLATFORM_ARG --target cni-binaries \

diff --git a/build/images/ovs/build.sh b/build/images/ovs/build.sh
@@ -90,8 +90,21 @@ OVS_VERSION=$(head -n 1 ../deps/ovs-version)
 
 if $PULL; then
     docker pull $PLATFORM_ARG ubuntu:20.04
-    docker pull $PLATFORM_ARG antrea/openvswitch-debs:$OVS_VERSION || true
-    docker pull $PLATFORM_ARG antrea/openvswitch:$OVS_VERSION || true
+    IMAGES_LIST=(
+        "antrea/openvswitch-debs:$OVS_VERSION"
+        "antrea/openvswitch:$OVS_VERSION"
+    )
+    for image in "${IMAGES_LIST[@]}"; do
+        if [[ ${DOCKER_REGISTRY} == "" ]]; then
+            docker pull $PLATFORM_ARG "${image}" || true
+        else
+            rc=0
+            docker pull "${DOCKER_REGISTRY}/${image}" || rc=$?
+            if [[ $rc -eq 0 ]]; then
+                docker tag "${DOCKER_REGISTRY}/${image}" "${image}"
+            fi
+        fi
+    done
 fi
 
 docker build $PLATFORM_ARG --target ovs-debs \

diff --git a/ci/jenkins/jobs/projects.yaml b/ci/jenkins/jobs/projects.yaml
@@ -166,7 +166,7 @@
           wrappers:
           - timeout:
               fail: true
-              timeout: 10
+              timeout: 20
               type: absolute
           - credentials-binding:
               - text:
@@ -208,7 +208,7 @@
           wrappers:
           - timeout:
               fail: true
-              timeout: 10
+              timeout: 20
               type: absolute
           - credentials-binding:
               - text:

diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh
@@ -135,6 +135,13 @@ if [[ "$WORKDIR" != "$DEFAULT_WORKDIR" && "$KUBECONFIG_PATH" == "$DEFAULT_KUBECO
     KUBECONFIG_PATH=$WORKDIR/.kube/config
 fi
 
+# If DOCKER_REGISTRY is non null, we ensure that "make" commands never pull from docker.io.
+NO_PULL=
+if [[ ${DOCKER_REGISTRY} != "" ]]; then
+    NO_PULL=1
+fi
+export NO_PULL
+
 function saveLogs() {
     echo "=== Truncate old logs ==="
     mkdir -p $WORKDIR/antrea_logs
@@ -284,24 +291,22 @@ function deliver_antrea {
     # because they might be being used in other builds running simultaneously.
     docker image prune -f --filter "until=1h" || true > /dev/null
     cd $GIT_CHECKOUT_DIR
-    if [[ ${DOCKER_REGISTRY} != "" ]]; then
-        docker pull ${DOCKER_REGISTRY}/antrea/base-ubuntu:2.14.0
-        docker tag ${DOCKER_REGISTRY}/antrea/base-ubuntu:2.14.0 antrea/base-ubuntu:2.14.0
-        docker pull ${DOCKER_REGISTRY}/antrea/golang:1.15
-        docker tag ${DOCKER_REGISTRY}/antrea/golang:1.15 golang:1.15
-    fi
+    # Ensure that files in the Docker context have the correct permissions, or Docker caching cannot
+    # be leveraged successfully
+    chmod -R g-w build/images/ovs
+    chmod -R g-w build/images/base
     for i in `seq 2`
     do
         if [[ "$COVERAGE" == true ]]; then
-            VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make build-ubuntu-coverage && break
+            VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-ubuntu-all.sh --pull --coverage && break
         else
-            VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make && break
+            VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-ubuntu-all.sh --pull && break
         fi
     done
     if [[ "$COVERAGE" == true ]]; then
-      VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make flow-aggregator-ubuntu-coverage
+      VERSION="$CLUSTER" make flow-aggregator-ubuntu-coverage
     else
-      VERSION="$CLUSTER" DOCKER_REGISTRY="${DOCKER_REGISTRY}" make flow-aggregator-image
+      VERSION="$CLUSTER" make flow-aggregator-image
     fi
     cd ci/jenkins
 
@@ -396,12 +401,9 @@ function run_integration {
     VM_IP=$(govc vm.ip ${VM_NAME}) # wait for VM to be on
 
     set -x
-    echo "===== Run Integration test ====="
-    if [[ ${DOCKER_REGISTRY} != "" ]]; then
-        docker pull ${DOCKER_REGISTRY}/antrea/openvswitch:2.14.0
-        docker tag ${DOCKER_REGISTRY}/antrea/openvswitch:2.14.0 antrea/openvswitch:2.14.0
-    fi
-    ${SSH_WITH_UTILS_KEY} -n jenkins@${VM_IP} "git clone ${ghprbAuthorRepoGitUrl} antrea && cd antrea && git checkout ${GIT_BRANCH} && DOCKER_REGISTRY=${DOCKER_REGISTRY} make docker-test-integration"
+    echo "===== Run Integration tests ====="
+    # umask ensures that files are cloned with the correct permissions so that Docker caching can be leveraged
+    ${SSH_WITH_UTILS_KEY} -n jenkins@${VM_IP} "umask 0022 && git clone ${ghprbAuthorRepoGitUrl} antrea && cd antrea && git checkout ${GIT_BRANCH} && DOCKER_REGISTRY=${DOCKER_REGISTRY} ./build/images/ovs/build.sh --pull && NO_PULL=${NO_PULL} make docker-test-integration"
     if [[ "$COVERAGE" == true ]]; then
         ${SSH_WITH_UTILS_KEY} -n jenkins@${VM_IP} "curl -s https://codecov.io/bash | bash -s -- -c -t ${CODECOV_TOKEN} -F integration-tests -f '.coverage/coverage-integration.txt'"
     fi

diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
@@ -93,22 +93,18 @@ done
 if [[ "$WORKDIR" != "$DEFAULT_WORKDIR" && "$KUBECONFIG_PATH" == "$DEFAULT_KUBECONFIG_PATH" ]]; then
     KUBECONFIG_PATH=${WORKDIR}/.kube/config
 fi
+NO_PULL=
 if [[ "$DOCKER_REGISTRY" != "" ]]; then
     # Image pulling policy of Sonobuoy is 'Always' by default. With dockerhub rate limit, sometimes it is better to use
     # cache when registry is used.
     IMAGE_PULL_POLICY="IfNotPresent"
+    # If DOCKER_REGISTRY is non null, we ensure that "make" commands never pull from docker.io.
+    NO_PULL=1
 fi
+export NO_PULL
 
 E2ETEST_PATH=${WORKDIR}/kubernetes/_output/dockerized/bin/linux/amd64/e2e.test
 
-function pull_antrea_ubuntu_image {
-    harbor_images=("base-ubuntu:2.14.0" "golang:1.15")
-    antrea_images=("antrea/base-ubuntu:2.14.0" "golang:1.15")
-    for i in {0..4}; do
-        docker pull ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]} || true
-    done
-}
-
 function export_govc_env_var {
     export GOVC_URL=$GOVC_URL
     export GOVC_USERNAME=$GOVC_USERNAME
@@ -187,10 +183,9 @@ function deliver_antrea_windows {
     make clean
     docker images | grep 'antrea-ubuntu' | awk '{print $3}' | xargs -r docker rmi -f || true
     docker images | grep '<none>' | awk '{print $3}' | xargs -r docker rmi || true
-    if [[ "$DOCKER_REGISTRY" != "" ]]; then
-        pull_antrea_ubuntu_image
-    fi
-    DOCKER_REGISTRY="${DOCKER_REGISTRY}" make
+    chmod -R g-w build/images/ovs
+    chmod -R g-w build/images/base
+    ./hack/build-antrea-ubuntu-all.sh --pull
     if [[ "$TESTCASE" =~ "networkpolicy" ]]; then
         make windows-bin
     fi
@@ -284,12 +279,13 @@ function deliver_antrea {
     docker images | grep 'antrea-ubuntu' | awk '{print $3}' | xargs -r docker rmi -f || true
     docker images | grep '<none>' | awk '{print $3}' | xargs -r docker rmi || true
     if [[ "${DOCKER_REGISTRY}" != "" ]]; then
-        pull_antrea_ubuntu_image
         docker pull "${DOCKER_REGISTRY}/antrea/sonobuoy-systemd-logs:v0.3"
         docker tag "${DOCKER_REGISTRY}/antrea/sonobuoy-systemd-logs:v0.3" "sonobuoy/systemd-logs:v0.3"
     fi
-    DOCKER_REGISTRY=${DOCKER_REGISTRY} make
-    DOCKER_REGISTRY=${DOCKER_REGISTRY} make flow-aggregator-image
+    chmod -R g-w build/images/ovs
+    chmod -R g-w build/images/base
+    ./hack/build-antrea-ubuntu-all.sh --pull
+    make flow-aggregator-image
 
     echo "====== Delivering Antrea to all the Nodes ======"
     echo "=== Fill serviceCIDRv6 and serviceCIDR ==="

diff --git a/hack/build-antrea-ubuntu-all.sh b/hack/build-antrea-ubuntu-all.sh
@@ -93,11 +93,33 @@ CNI_BINARIES_VERSION=$(head -n 1 build/images/deps/cni-binaries-version)
 # image! This is a bit more inconvenient to maintain, but we rarely introduce
 # new base images in the build chain.
 if $PULL; then
+    # Always pull ubuntu:20.04 from DockerHub even if DOCKER_REGISTRY is set, as
+    # it is not an Antrea image and there should be no rate-limiting for
+    # Canonical images.
     docker pull $PLATFORM_ARG ubuntu:20.04
-    docker pull $PLATFORM_ARG antrea/openvswitch-debs:$OVS_VERSION || true
-    docker pull $PLATFORM_ARG antrea/openvswitch:$OVS_VERSION || true
-    docker pull $PLATFORM_ARG antrea/cni-binaries:$CNI_BINARIES_VERSION || true
-    docker pull $PLATFORM_ARG antrea/base-ubuntu:$OVS_VERSION || true
+    if [[ ${DOCKER_REGISTRY} == "" ]]; then
+        docker pull $PLATFORM_ARG golang:1.15
+    else
+        docker pull ${DOCKER_REGISTRY}/antrea/golang:1.15
+        docker tag ${DOCKER_REGISTRY}/antrea/golang:1.15 golang:1.15
+    fi
+    IMAGES_LIST=(
+        "antrea/openvswitch-debs:$OVS_VERSION"
+        "antrea/openvswitch:$OVS_VERSION"
+        "antrea/cni-binaries:$CNI_BINARIES_VERSION"
+        "antrea/base-ubuntu:$OVS_VERSION"
+    )
+    for image in "${IMAGES_LIST[@]}"; do
+        if [[ ${DOCKER_REGISTRY} == "" ]]; then
+            docker pull $PLATFORM_ARG "${image}" || true
+        else
+            rc=0
+            docker pull "${DOCKER_REGISTRY}/${image}" || rc=$?
+            if [[ $rc -eq 0 ]]; then
+                docker tag "${DOCKER_REGISTRY}/${image}" "${image}"
+            fi
+        fi
+    done
 fi
 
 cd build/images/ovs