new build script to actually build the docker amd image

.buildkite/dev-docker/pipeline.yml

@@ -1,3 +1,13 @@
 steps:
-  - label: ":wave: Greetings"            # Label (with rich emojis https://ela.st/bk-emoji).
-    command: "echo 'My first pipeline from a branch!'" # Command to run (evaluated by Bash).
+  - input: "Build parameters"
+    fields:
+      - text: "BUILD_FROM_BRANCH"
+        key: "BUILD_FROM_BRANCH"
+        default: "master"
+        hint: "The branch to build from e.g. 'master'."
+  - wait
+  - label: "Release Docker Artifacts for Rally"
+    command: bash .buildkite/docker-dev/run.sh
+    # Run on GCP to use `docker`
+    agents:
+      provider: gcp

.buildkite/dev-docker/run.sh

@@ -5,7 +5,8 @@ set -eo pipefail
 source .buildkite/retry.sh
 
 set +x
-RELEASE_VERSION=$(buildkite-agent meta-data get RELEASE_VERSION)
+BUILD_FROM_BRANCH=$(buildkite-agent meta-data get RELEASE_VERSION)
+
 # login to docker registry
 DOCKER_PASSWORD=$(vault read -field token /secret/ci/elastic-rally/release/docker-hub-rally)
 retry 5 docker login -u elasticmachine -p $DOCKER_PASSWORD
@@ -15,13 +16,13 @@ tmp_dir=$(mktemp --directory)
 pushd "$tmp_dir"
 git clone https://github.com/elastic/rally
 pushd rally
-git checkout "${RELEASE_VERSION}"
+git checkout "${BUILD_FROM_BRANCH}"
 git --no-pager show
 
 set -x
 export TERM=dumb
 export LC_ALL=en_US.UTF-8
-./release-docker.sh "$RELEASE_VERSION"
+./release-docker.sh "$BUILD_FROM_BRANCH"
 
 popd
 popd

build-dev-docker.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Prerequisites for releasing:
+
+# Logged in on Docker Hub (docker login)
+
+# fail this script immediately if any command fails with a non-zero exit code
+set -eu -eE -o functrace
+
+function push_failed {
+    echo "Error while pushing Docker image. Did you \`docker login\`?"
+}
+
+if [[ $# -eq 0 ]] ; then
+    echo "ERROR: $0 requires the Rally version to push as a command line argument and you didn't supply it."
+    echo "For example: $0 master"
+    exit 1
+fi
+export RALLY_BRANCH=$1
+export RALLY_LICENSE=$(awk 'FNR>=2 && FNR<=2' LICENSE | sed 's/^[ \t]*//')
+
+export GIT_SHA=$(git rev-parse --short HEAD)
+export DATE=$(date +%Y%m%d)
+
+export RALLY_VERSION="${RALLY_BRANCH}-${GIT_SHA}-${DATE}"
+export MAIN_BRANCH=$(git remote show origin | sed -n '/HEAD branch/s/.*: //p')
+
+if [[ $RALLY_BRANCH == $MAIN_BRANCH ]]; then
+    export DOCKER_TAG_LATEST="dev-latest"
+else
+    export DOCKER_TAG_LATEST="${RALLY_BRANCH}-latest"
+fi
+
+echo "========================================================"
+echo "Building Docker image for Rally $RALLY_VERSION          "
+echo "========================================================"
+
+docker build -t elastic/rally:${RALLY_VERSION} --build-arg RALLY_VERSION --build-arg RALLY_LICENSE -f docker/Dockerfiles/Dockerfile-dev $PWD
+
+echo "======================================================="
+echo "Testing Docker image for Rally release $RALLY_VERSION  "
+echo "======================================================="
+
+./release-docker-test.sh dev
+
+echo "======================================================="
+echo "Publishing Docker image elastic/rally:$RALLY_VERSION   "
+echo "======================================================="
+
+trap push_failed ERR
+docker push elastic/rally:${RALLY_VERSION}
+
+echo "============================================"
+echo "Publishing Docker image elastic/rally:latest"
+echo "============================================"
+
+docker tag elastic/rally:${RALLY_VERSION} elastic/rally:${DOCKER_TAG_LATEST}
+docker push elastic/rally:${DOCKER_TAG_LATEST}
+
+trap - ERR

release-docker-test.sh

@@ -101,7 +101,11 @@ function test_docker_release_image {
     docker_compose down
 
     info "Testing Rally docker image uses the right version"
-    actual_version=$(docker run --rm elastic/rally:${RALLY_VERSION} esrally --version | cut -d ' ' -f 2,2)
+    if [[ "${DEVELOPMENT}" == "YES" ]]; then
+        actual_version=${RALLY_VERSION}
+    else
+        actual_version=$(docker run --rm elastic/rally:${RALLY_VERSION} esrally --version | cut -d ' ' -f 2,2)
+    fi
     if [[ ${actual_version} != ${RALLY_VERSION} ]]; then
         echo "Rally version in Docker image: [${actual_version}] doesn't match the expected version [${RALLY_VERSION}]"
         exit 1
@@ -159,4 +163,10 @@ function tear_down {
 
 trap "tear_down" EXIT
 
+if [[ $# -gt 0 && $1 -eq "dev" ]] ; then
+    export DEVELOPMENT="YES"
+else 
+    export DEVELOPMENT="NO"
+fi
+
 main