Enable envoy images build on Arm CI environments (#11813)

In this patch, it will enable the envoyproxy/envoy arm image to build
in community arm CI environments.
1. Do some modifications in docker_ci.sh script for building arm images
   by buildx. It will firstly set up environments. Then use the buildx
   tool to build the envoyproxy/envoy arm images on x86 platform.
2. Modify the docker build job for building multi-arch images.
   It will firstly download the arm64 and amd64 envoy binaries. Then
   invoke the docker_ci.sh scripts to generate images.

Risk Level: Medium (of breaking images)
Testing: CI
Docs Changes: N/A
Release Notes: Added
Fixes #1861 

Signed-off-by: Jingzhao.Ni <[email protected]>

.azure-pipelines/pipelines.yml

@@ -122,8 +122,8 @@ jobs:
         condition: always()
 
   - job: docker
-    displayName: "Linux-x64 docker"
-    dependsOn: ["release"]
+    displayName: "Linux multi-arch docker"
+    dependsOn: ["release","release_arm64"]
     condition: and(succeeded(), eq(variables['PostSubmit'], 'true'), ne(variables['Build.Reason'], 'PullRequest'))
     pool:
       vmImage: "ubuntu-18.04"
@@ -135,10 +135,17 @@ jobs:
           itemPattern: "bazel.release/envoy_binary.tar.gz"
           downloadType: single
           targetPath: $(Build.StagingDirectory)
-
+      - task: DownloadBuildArtifacts@0
+        inputs:
+          buildType: current
+          artifactName: "bazel.release.arm64"
+          itemPattern: "bazel.release.arm64/envoy_binary.tar.gz"
+          downloadType: single
+          targetPath: $(Build.StagingDirectory)
       - bash: |
           set -e
-          tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz
+          mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64
+          mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64
           ci/docker_ci.sh
         workingDirectory: $(Build.SourcesDirectory)
         env:

ci/Dockerfile-envoy

@@ -1,6 +1,5 @@
 ARG BUILD_FROM=ubuntu:18.04
 
-
 # Build stage
 FROM $BUILD_FROM as build
 
@@ -17,7 +16,7 @@ RUN apt-get update \
 
 # Final stage
 FROM $BUILD_FROM
-
+ARG TARGETPLATFORM
 RUN apt-get update \
     && apt-get upgrade -y \
     && apt-get install -y ca-certificates \
@@ -31,7 +30,7 @@ RUN adduser --group --system envoy
 
 RUN mkdir -p /etc/envoy
 
-ADD build_release_stripped/envoy /usr/local/bin/envoy
+ADD ${TARGETPLATFORM}/build_release_stripped/envoy /usr/local/bin/envoy
 ADD configs/google_com_proxy.v2.yaml /etc/envoy/envoy.yaml
 
 EXPOSE 10000

ci/Dockerfile-envoy-alpine

@@ -1,8 +1,8 @@
 FROM frolvlad/alpine-glibc:alpine-3.12_glibc-2.31
-
 RUN mkdir -p /etc/envoy
 
-ADD build_release_stripped/envoy /usr/local/bin/envoy
+ADD linux/amd64/build_release_stripped/envoy /usr/local/bin/envoy
+
 ADD configs/google_com_proxy.v2.yaml /etc/envoy/envoy.yaml
 RUN apk add --no-cache shadow su-exec \
         && addgroup -S envoy && adduser --no-create-home -S envoy -G envoy

ci/Dockerfile-envoy-alpine-debug

@@ -1,8 +1,7 @@
 FROM frolvlad/alpine-glibc:alpine-3.12_glibc-2.31
-
 RUN mkdir -p /etc/envoy
 
-ADD build_release/envoy /usr/local/bin/envoy
+ADD linux/amd64/build_release/envoy /usr/local/bin/envoy
 ADD configs/google_com_proxy.v2.yaml /etc/envoy/envoy.yaml
 RUN apk add --no-cache shadow su-exec \
         && addgroup -S envoy && adduser --no-create-home -S envoy -G envoy

ci/docker_ci.sh

@@ -4,16 +4,59 @@
 # CI logs.
 set -e
 
+# Setting environments for buildx tools
+config_env(){
+    # Qemu configurations
+    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+    # Remove older build instance
+    docker buildx rm  multi-builder | true
+    docker buildx create --use --name multi-builder --platform linux/arm64,linux/amd64
+}
+
+build_images(){
+    TYPE=$1
+    BUILD_TAG=$2
+
+    # Only build/push envoyproxy/envoy multi-arch images since others still do not support.
+    if [ -z "${TYPE}" ]; then
+        docker buildx build --platform linux/arm64 -f ci/Dockerfile-envoy"${TYPE}" -t ${BUILD_TAG} .
+        # Export envoyproxy/envoy amd64 image which will be used for building envoyproxy/envoy-google-vrp
+        docker buildx build --platform linux/amd64 -f ci/Dockerfile-envoy"${TYPE}" -o type=docker -t ${BUILD_TAG} .
+    elif [ "${TYPE}" == "-google-vrp" ]; then
+        # The envoyproxy/envoy-google-vrp is based on envoyproxy/envoy image. So it is built from cache envoyproxy/envoy:local
+        docker build -f ci/Dockerfile-envoy"${TYPE}" --cache-from "${DOCKER_IMAGE_PREFIX}:local" -t ${BUILD_TAG} .
+    else
+        docker build -f ci/Dockerfile-envoy"${TYPE}" -t ${BUILD_TAG} .
+    fi
+}
+
+push_images(){
+    TYPE=$1
+    BUILD_TAG=$2
+
+    if [ -z "${TYPE}" ]; then
+        # Only push envoyproxy/envoy multi-arch images since others still do not support.
+        docker buildx build --platform linux/arm64,linux/amd64 --push -f ci/Dockerfile-envoy"${TYPE}" -t ${BUILD_TAG} .
+    else
+        docker tag "${DOCKER_IMAGE_PREFIX}${TYPE}:local" ${BUILD_TAG}
+        docker push ${BUILD_TAG}
+    fi
+}
+
 # This prefix is altered for the private security images on setec builds.
 DOCKER_IMAGE_PREFIX="${DOCKER_IMAGE_PREFIX:-envoyproxy/envoy}"
 
 # "-google-vrp" must come afer "" to ensure we rebuild the local base image dependency.
 BUILD_TYPES=("" "-alpine" "-alpine-debug" "-google-vrp")
 
+# Configure docker-buildx tools
+config_env
+
 # Test the docker build in all cases, but use a local tag that we will overwrite before push in the
 # cases where we do push.
 for BUILD_TYPE in "${BUILD_TYPES[@]}"; do
-    docker build -f ci/Dockerfile-envoy"${BUILD_TYPE}" -t "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}:local" .
+    build_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}:local"
 done
 
 MASTER_BRANCH="refs/heads/master"
@@ -42,21 +85,16 @@ fi
 docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD"
 
 for BUILD_TYPE in "${BUILD_TYPES[@]}"; do
-    docker tag "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}:local" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${IMAGE_NAME}"
-    docker push "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${IMAGE_NAME}"
+    push_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${IMAGE_NAME}"
 
     # Only push latest on master builds.
     if [[ "${AZP_BRANCH}" == "${MASTER_BRANCH}" ]]; then
-        docker tag "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}:local" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:latest"
-        docker push "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:latest"
+        push_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:latest"
     fi
 
     # Push vX.Y-latest to tag the latest image in a release line
     if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
       RELEASE_LINE=$(echo "$IMAGE_NAME" | sed -E 's/(v[0-9]+\.[0-9]+)\.[0-9]+/\1-latest/')
-      docker tag "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}:local" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${RELEASE_LINE}"
-      docker push "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${RELEASE_LINE}"
+        push_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${RELEASE_LINE}"
     fi
 done
-
-

docs/root/version_history/current.rst

@@ -41,6 +41,7 @@ New Features
 ------------
 
 * access log: added support for :ref:`%DOWNSTREAM_PEER_FINGERPRINT_1% <config_access_log_format_response_flags>` as a response flag.
+* build: enable building envoy arm64 images by buildx tool in x86 CI platform.
 * dynamic_forward_proxy: added :ref:`use_tcp_for_dns_lookups<envoy_v3_api_field_extensions.common.dynamic_forward_proxy.v3.DnsCacheConfig.use_tcp_for_dns_lookups>` option to use TCP for DNS lookups in order to match the DNS options for :ref:`Clusters<envoy_v3_api_msg_config.cluster.v3.Cluster>`.
 * ext_authz filter: added support for emitting dynamic metadata for both :ref:`HTTP <config_http_filters_ext_authz_dynamic_metadata>` and :ref:`network <config_network_filters_ext_authz_dynamic_metadata>` filters.
 * grpc-json: support specifying `response_body` field in for `google.api.HttpBody` message.