feat: add release build for daily docker builds

.buildkite/release-docker/manifest.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+source .buildkite/retry.sh
+
+set +x
+
+RELEASE_VERSION=$(buildkite-agent meta-data get RELEASE_VERSION)
+
+# login to docker registry
+DOCKER_PASSWORD=$(vault read -field token /secret/ci/elastic-rally/release/docker-hub-rally)
+retry 5 docker login -u elasticmachine -p $DOCKER_PASSWORD
+unset DOCKER_PASSWORD
+
+set -x
+export TERM=dumb
+export LC_ALL=en_US.UTF-8
+./release-docker-manifest.sh "$RELEASE_VERSION"

.buildkite/release-docker/pipeline.yml

@@ -1,3 +1,7 @@
+agents:
+  provider: "gcp"
+  zone: "us-central1-a"
+
 steps:
   - input: "Build parameters"
     fields:
@@ -8,7 +12,22 @@ steps:
 
   - wait
   - label: "Release Docker Artifacts for Rally"
-    command: bash .buildkite/release-docker/run.sh
+    command: bash .buildkite/release-docker/run.sh amd64
+    # Run on GCP to use `docker`
+    key: "amd64"
+    agents:
+      machineType: "n2-standard-8"
+      image: family/core-ubuntu-2204
+  - label: "Release Docker Artifacts for Rally"
+    command: bash .buildkite/release-docker/run.sh arm64
     # Run on GCP to use `docker`
+    key: "arm64"
     agents:
-      provider: gcp
+      machineType: "t2a-standard-8"
+      image: family/core-ubuntu-2204-aarch64
+  - label: ":docker: build docker manifest"
+    command: bash .buildkite/release-docker/manifest.sh
+    key: "manifest"
+    depends_on:
+      - "amd64"
+      - "arm64"

.buildkite/release-docker/run.sh

@@ -6,6 +6,14 @@ source .buildkite/retry.sh
 
 set +x
 RELEASE_VERSION=$(buildkite-agent meta-data get RELEASE_VERSION)
+
+if [[ $# -ne 1 ]]; then
+    echo "Usage: $0 amd64"
+    exit 1
+fi
+
+ARCH="$1"
+
 # login to docker registry
 DOCKER_PASSWORD=$(vault read -field token /secret/ci/elastic-rally/release/docker-hub-rally)
 retry 5 docker login -u elasticmachine -p $DOCKER_PASSWORD
@@ -21,7 +29,7 @@ git --no-pager show
 set -x
 export TERM=dumb
 export LC_ALL=en_US.UTF-8
-./release-docker.sh "$RELEASE_VERSION"
+./release-docker.sh "$RELEASE_VERSION" "$ARCH"
 
 popd
 popd

build-dev-docker-manifest.sh

@@ -22,7 +22,7 @@
 # Logged in on Docker Hub (docker login)
 
 # fail this script immediately if any command fails with a non-zero exit code
-set -eu -eE -o functrace
+set -eu
 
 function push_failed {
     echo "Error while pushing Docker image. Did you \`docker login\`?"

build-dev-docker.sh

@@ -22,7 +22,7 @@
 # Logged in on Docker Hub (docker login)
 
 # fail this script immediately if any command fails with a non-zero exit code
-set -eu -eE -o functrace
+set -eu
 
 function push_failed {
     echo "Error while pushing Docker image. Did you \`docker login\`?"
@@ -41,6 +41,7 @@ export GIT_SHA=$(git rev-parse --short HEAD)
 export DATE=$(date +%Y%m%d)
 
 export RALLY_VERSION="${RALLY_BRANCH}-${GIT_SHA}-${DATE}-${ARCH}"
+export RALLY_VERSION_TAG="${RALLY_VERSION}"
 export MAIN_BRANCH=$(git remote show origin | sed -n '/HEAD branch/s/.*: //p')
 
 if [[ $RALLY_BRANCH == $MAIN_BRANCH ]]; then

docker/docker-compose-tests.yml

@@ -1,7 +1,7 @@
 version: '2.2'
 services:
   rally:
-    image: elastic/rally:${RALLY_VERSION}
+    image: elastic/rally:${RALLY_VERSION_TAG}
     container_name: rally
     command: ${TEST_COMMAND}
     volumes:

release-docker-manifest.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Prerequisites for releasing:
+
+# Logged in on Docker Hub (docker login)
+
+# fail this script immediately if any command fails with a non-zero exit code
+set -eu
+
+function push_failed {
+    echo "Error while pushing Docker image. Did you \`docker login\`?"
+}
+
+if [[ $# -eq 0 ]] ; then
+    echo "ERROR: $0 requires the Rally version to push as a command line argument and you didn't supply it."
+    echo "For example: $0 master"
+    exit 1
+fi
+export RALLY_VERSION=$1
+export RALLY_LICENSE=$(awk 'FNR>=2 && FNR<=2' LICENSE | sed 's/^[ \t]*//')
+
+export GIT_SHA=$(git rev-parse --short HEAD)
+export DATE=$(date +%Y%m%d)
+
+export RALLY_VERSION_TAG="${RALLY_VERSION}-${DATE}"
+export DOCKER_TAG_LATEST="latest"
+
+echo "========================================================"
+echo "Pulling Docker images for Rally $RALLY_VERSION_TAG          "
+echo "========================================================"
+
+docker pull elastic/rally:${RALLY_VERSION_TAG}-amd64
+docker pull elastic/rally:${RALLY_VERSION_TAG}-arm64
+
+echo "======================================================="
+echo "Creating Docker manifest image for Rally $RALLY_VERSION_TAG"
+echo "======================================================="
+
+docker manifest create elastic/rally:${RALLY_VERSION_TAG} \
+    --amend elastic/rally:${RALLY_VERSION_TAG}-amd64 \
+    --amend elastic/rally:${RALLY_VERSION_TAG}-arm64
+
+echo "======================================================="
+echo "Creating Docker manifest image for Rally $DOCKER_TAG_LATEST"
+echo "======================================================="
+
+docker manifest create elastic/rally:${DOCKER_TAG_LATEST} \
+    --amend elastic/rally:${DOCKER_TAG_LATEST}-amd64 \
+    --amend elastic/rally:${DOCKER_TAG_LATEST}-arm64
+
+echo "======================================================="
+echo "Publishing Docker image elastic/rally:$RALLY_VERSION_TAG"
+echo "======================================================="
+
+trap push_failed ERR
+docker manifest push elastic/rally:${RALLY_VERSION_TAG}
+docker manifest push elastic/rally:${DOCKER_TAG_LATEST}
+
+trap - ERR
+

release-docker-test.sh

@@ -104,22 +104,22 @@ function test_docker_release_image {
     if [[ "${DEVELOPMENT}" == "YES" ]]; then
         actual_version=${RALLY_VERSION}
     else
-        actual_version=$(docker run --rm elastic/rally:${RALLY_VERSION} esrally --version | cut -d ' ' -f 2,2)
+        actual_version=$(docker run --rm elastic/rally:${RALLY_VERSION_TAG} esrally --version | cut -d ' ' -f 2,2)
     fi
     if [[ ${actual_version} != ${RALLY_VERSION} ]]; then
         echo "Rally version in Docker image: [${actual_version}] doesn't match the expected version [${RALLY_VERSION}]"
         exit 1
     fi
 
     info "Testing Rally docker image version label is correct"
-    actual_version=$(docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' elastic/rally:${RALLY_VERSION})
+    actual_version=$(docker inspect --format '{{ index .Config.Labels "org.label-schema.version"}}' elastic/rally:${RALLY_VERSION_TAG})
     if [[ ${actual_version} != ${RALLY_VERSION} ]]; then
         echo "org.label-schema.version label in Rally Docker image: [${actual_version}] doesn't match the expected version [${RALLY_VERSION}]"
         exit 1
     fi
 
     info "Testing Rally docker image license label is correct"
-    actual_license=$(docker inspect --format '{{ index .Config.Labels "license"}}' elastic/rally:${RALLY_VERSION})
+    actual_license=$(docker inspect --format '{{ index .Config.Labels "license"}}' elastic/rally:${RALLY_VERSION_TAG})
     if [[ ${actual_license} != ${RALLY_LICENSE} ]]; then
         echo "license label in Rally Docker image: [${actual_license}] doesn't match the expected license [${RALLY_LICENSE}]"
         exit 1

release-docker.sh

@@ -28,38 +28,43 @@ function push_failed {
     echo "Error while pushing Docker image. Did you \`docker login\`?"
 }
 
-if [[ $# -eq 0 ]] ; then
-    echo "ERROR: $0 requires the Rally version to push as a command line argument and you didn't supply it."
-    echo "For example: $0 1.1.0"
+if [[ $# -ne 2 ]] ; then
+    echo "ERROR: $0 requires the Rally version and architecture to push as a command line argument and you didn't supply it."
+    echo "For example: $0 1.1.0 amd64"
     exit 1
 fi
 export RALLY_VERSION=$1
+export ARCH=$2
 export RALLY_LICENSE=$(awk 'FNR>=2 && FNR<=2' LICENSE | sed 's/^[ \t]*//')
+export DATE=$(date +%Y%m%d)
+
+export RALLY_VERSION_TAG="${RALLY_VERSION}-${DATE}-${ARCH}"
+export DOCKER_TAG_LATEST="latest-${ARCH}"
 
 echo "========================================================"
-echo "Building Docker image for Rally release $RALLY_VERSION  "
+echo "Building Docker image Rally release $RALLY_VERSION_TAG  "
 echo "========================================================"
 
-docker build -t elastic/rally:${RALLY_VERSION} --build-arg RALLY_VERSION --build-arg RALLY_LICENSE -f docker/Dockerfiles/release/Dockerfile $PWD
+docker build -t elastic/rally:${RALLY_VERSION_TAG} --build-arg RALLY_VERSION --build-arg RALLY_LICENSE -f docker/Dockerfiles/release/Dockerfile $PWD
 
 echo "======================================================="
-echo "Testing Docker image for Rally release $RALLY_VERSION  "
+echo "Testing Docker image Rally release $RALLY_VERSION_TAG  "
 echo "======================================================="
 
 ./release-docker-test.sh
 
 echo "======================================================="
-echo "Publishing Docker image elastic/rally:$RALLY_VERSION   "
+echo "Publishing Docker image elastic/rally:$RALLY_VERSION_TAG"
 echo "======================================================="
 
 trap push_failed ERR
-docker push elastic/rally:${RALLY_VERSION}
+docker push elastic/rally:${RALLY_VERSION_TAG}
 
 echo "============================================"
 echo "Publishing Docker image elastic/rally:latest"
 echo "============================================"
 
-docker tag elastic/rally:${RALLY_VERSION} elastic/rally:latest
-docker push elastic/rally:latest
+docker tag elastic/rally:${RALLY_VERSION_TAG} elastic/rally:${DOCKER_TAG_LATEST}
+docker push elastic/rally:${DOCKER_TAG_LATEST}
 
 trap - ERR