From 509fa1d39e2275b04f801cf54ae70073ba6899bc Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 10 Mar 2020 12:44:14 +0100 Subject: [PATCH] [AIRFLOW-7015] Detect Dockerhub repo/user when building on Dockerhub (#7673) (cherry picked from commit e2711af83d286f0b780e24e50457b0e8a5e538df) --- common/_autodetect_variables.sh | 104 ------------------ common/_image_variables.sh | 35 ++++++ scripts/ci/_utils.sh | 4 +- scripts/ci/ci_build_dockerhub.sh | 23 +++- .../in_container/kubernetes/app/deploy_app.sh | 4 +- 5 files changed, 61 insertions(+), 109 deletions(-) delete mode 100644 common/_autodetect_variables.sh create mode 100644 common/_image_variables.sh diff --git a/common/_autodetect_variables.sh b/common/_autodetect_variables.sh deleted file mode 100644 index ecab63427f75c6..00000000000000 --- a/common/_autodetect_variables.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This autodetection of PYTHON_VERSION works in this sequence: -# -# 1) When running builds on DockerHub we cannot pass different environment -# variables there, so we detect python version based on the image -# name being build (airflow:master-python3.7 -> PYTHON_VERSION=3.7). -# DockerHub adds index.docker.io in front of the image name. -# -# 2) When running builds on Travis CI. We use python version determined -# from default python3 version available on the path. This way we -# do not have to specify PYTHON_VERSION separately in each job. -# We just specify which host python version is used for that job. -# This makes a nice UI experience where you see python version in -# Travis UI. -# -# 3) Running builds locally via scripts where we can pass PYTHON_VERSION -# as environment variable. -# -# 4) Running builds locally for the first time with Breeze. By default -# we determine the version based on default python3 version we have -# in the host system (3.5, 3.6 or 3.7) and we use this one. -# -# 5) Selecting python version with Breeze's --python switch. This will -# override python version but it will also store the last used version -# of python in .build directory so that it is automatically used next -# time. - -# shellcheck source=common/_default_branch.sh -. "${AIRFLOW_SOURCES}/common/_default_branch.sh" - -# You can override DOCKERHUB_USER to use your own DockerHub account and play with your -# own docker images. In this case you can build images locally and push them -export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} - -# You can override DOCKERHUB_REPO to use your own DockerHub repository and play with your -# own docker images. In this case you can build images locally and push them -export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} - -# if BRANCH_NAME is not set it will be set to either SOURCE_BRANCH -# (if overridden by configuration) or default branch configured in /common/_default_branch.sh -export SOURCE_BRANCH=${SOURCE_BRANCH:=${DEFAULT_BRANCH}} - -export BRANCH_NAME=${BRANCH_NAME:=${SOURCE_BRANCH}} - -# IMAGE_NAME might come from DockerHub and we decide which PYTHON_VERSION to use based on it (see below) -# In case IMAGE_NAME is not set we determine PYTHON_VERSION based on the default python on the path -# So in virtualenvs it will use the virtualenv's PYTHON_VERSION and in DockerHub it will determine -# PYTHON_VERSION from the IMAGE_NAME set in the DockerHub build. -# See comment above in PYTHON_VERSION - we will be able to get rid of this cumbersomness when we switch -# to a different CI system and start pushing images to DockerHub rather than build it there. -export BASE_IMAGE_NAME=${IMAGE_NAME:=${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_VERSION:-3.5}} - -# Remove index.docker.io/ prefix as it is added by default by DockerHub -export LOCAL_BASE_IMAGE_NAME=${BASE_IMAGE_NAME#index.docker.io/} - -if [[ ! ${LOCAL_BASE_IMAGE_NAME} == ${DOCKERHUB_USER}/${DOCKERHUB_REPO}* ]]; then - echo >&2 - echo >&2 "ERROR: The ${LOCAL_BASE_IMAGE_NAME} does not start with ${DOCKERHUB_USER}/${DOCKERHUB_REPO}" - echo >&2 - exit 1 -fi - -# Extract IMAGE_TAG from LOCAL_BASE_IMAGE_NAME (apache/airflow:master-python3.6 -> master-python3.6) -export IMAGE_TAG=${LOCAL_BASE_IMAGE_NAME##${DOCKERHUB_USER}/${DOCKERHUB_REPO}:} - -# Extract TAG_PREFIX from IMAGE_TAG (master-python3.6 -> master) -export TAG_PREFIX=${IMAGE_TAG%-*} - -# By that time no matter if we got the version from the IMAGE_NAME or we built -# IMAGE_NAME with PYTHON_VERSION - the LOCAL_BASE_IMAGE_NAME should end with the correct pythonX.Y -# So we Extract python version from image name. The bash construct below extracts last -# field after '-' delimiter, so for example 'airflow:master-python3.6' -# will produce LONG_PYTHON_VERSION_FROM_IMAGE=python3.6 -export LONG_PYTHON_VERSION_FROM_IMAGE="${LOCAL_BASE_IMAGE_NAME##*-}" - -# Verify that python version retrieved from image follows the expected pattern of python3.Y -if [[ ! ${LONG_PYTHON_VERSION_FROM_IMAGE} =~ python[23]\.[0-9]+ ]]; then - echo >&2 - echo >&2 "ERROR: Python version extracted from IMAGE_NAME does not match the python3.Y format" - echo >&2 - echo >&2 "The IMAGE_NAME format should be '-python[23].Y'" - echo >&2 - exit 1 -fi - -# Unless python_version is forced, read actual python version from IMAGE NAME -export PYTHON_VERSION=${LONG_PYTHON_VERSION_FROM_IMAGE#python} diff --git a/common/_image_variables.sh b/common/_image_variables.sh new file mode 100644 index 00000000000000..44f53ef386df53 --- /dev/null +++ b/common/_image_variables.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# shellcheck source=common/_default_branch.sh +. "${AIRFLOW_SOURCES}/common/_default_branch.sh" + +# You can override DOCKERHUB_USER to use your own DockerHub account and play with your +# own docker images. In this case you can build images locally and push them +export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} + +# You can override DOCKERHUB_REPO to use your own DockerHub repository and play with your +# own docker images. In this case you can build images locally and push them +export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} + +# if BRANCH_NAME is not set it will be set to either SOURCE_BRANCH +# (if overridden by configuration) or default branch configured in /common/_default_branch.sh +export SOURCE_BRANCH=${SOURCE_BRANCH:=${DEFAULT_BRANCH}} + +# read branch name from what has been set from sources (It can also be overridden) +export BRANCH_NAME=${BRANCH_NAME:=${SOURCE_BRANCH}} diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index c7e3fe34806021..9d1663f45882a9 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -53,8 +53,8 @@ function initialize_breeze_environment { mkdir -p "${AIRFLOW_SOURCES}/files" mkdir -p "${AIRFLOW_SOURCES}/dist" - # shellcheck source=common/_autodetect_variables.sh - . "${AIRFLOW_SOURCES}/common/_autodetect_variables.sh" + # shellcheck source=common/_image_variables.sh + . "${AIRFLOW_SOURCES}/common/_image_variables.sh" # shellcheck source=common/_files_for_rebuild_check.sh . "${AIRFLOW_SOURCES}/common/_files_for_rebuild_check.sh" diff --git a/scripts/ci/ci_build_dockerhub.sh b/scripts/ci/ci_build_dockerhub.sh index 58e7c0c110a8d1..8b4c7c847c452f 100755 --- a/scripts/ci/ci_build_dockerhub.sh +++ b/scripts/ci/ci_build_dockerhub.sh @@ -31,15 +31,36 @@ if [[ "${TRAVIS_EVENT_TYPE:=}" == "cron" ]]; then export PULL_BASE_IMAGES="true" fi +if [[ -z ${DOCKER_REPO} ]]; then + echo + echo "Error! Missing DOCKER_REPO environment variable" + echo "Please specify DOCKER_REPO variable following the pattern HOST/DOCKERHUB_USER/DOCKERHUB_REPO" + echo + exit 1 +else + echo "DOCKER_REPO=${DOCKER_REPO}" +fi + +[[ ${DOCKER_REPO:=} =~ [^/]*/([^/]*)/([^/]*) ]] && \ + export DOCKERHUB_USER=${BASH_REMATCH[1]} && + export DOCKERHUB_REPO=${BASH_REMATCH[2]} + +echo +echo "DOCKERHUB_USER=${DOCKERHUB_USER}" +echo "DOCKERHUB_REPO=${DOCKERHUB_REPO}" +echo + if [[ -z ${DOCKER_TAG:=} ]]; then echo echo "Error! Missing DOCKER_TAG environment variable" echo "Please specify DOCKER_TAG variable following the pattern BRANCH-pythonX.Y[-ci]" echo exit 1 +else + echo "DOCKER_TAG=${DOCKER_TAG}" fi -[[ ${DOCKER_TAG:=} =~ ${DEFAULT_BRANCH}-python([0-9.]*) ]] && export PYTHON_VERSION=${BASH_REMATCH[1]} +[[ ${DOCKER_TAG:=} =~ ${DEFAULT_BRANCH}-python([0-9.]*)(.*) ]] && export PYTHON_VERSION=${BASH_REMATCH[1]} if [[ -z ${PYTHON_VERSION:=} ]]; then echo diff --git a/scripts/ci/in_container/kubernetes/app/deploy_app.sh b/scripts/ci/in_container/kubernetes/app/deploy_app.sh index 36b41ee14f8594..de35db03ddf441 100755 --- a/scripts/ci/in_container/kubernetes/app/deploy_app.sh +++ b/scripts/ci/in_container/kubernetes/app/deploy_app.sh @@ -44,8 +44,8 @@ trap in_container_script_end EXIT export TEMPLATE_DIRNAME="${MY_DIR}/templates" export BUILD_DIRNAME="${MY_DIR}/build" -# shellcheck source=common/_autodetect_variables.sh -. "${AIRFLOW_SOURCES}/common/_autodetect_variables.sh" +# shellcheck source=common/_image_variables.sh +. "${AIRFLOW_SOURCES}/common/_image_variables.sh" # Source branch will be set in DockerHub SOURCE_BRANCH=${SOURCE_BRANCH:=${DEFAULT_BRANCH}}