[AIRFLOW-5828] Move build logic out from hooks/build (apache#7618)

This is the final step of simplifying the Breeze scripts by moving all the logic out from Travis' hooks/build (cherry picked from commit d18513c)
astronomer · Mar 19, 2020 · d5a1855 · d5a1855
1 parent d3602ee
commit d5a1855
Show file tree

Hide file tree

Showing 17 changed files with 455 additions and 568 deletions.
diff --git a/BREEZE.rst b/BREEZE.rst
@@ -364,7 +364,7 @@ Mounting Local Sources to Breeze
 Important sources of Airflow are mounted inside the ``airflow-testing`` container that you enter.
 This means that you can continue editing your changes on the host in your favourite IDE and have them
 visible in the Docker immediately and ready to test without rebuilding images. You can disable mounting
-by specifying ``--skip-mounting-source-volume`` flag when running Breeze. In this case you will have sources
+by specifying ``--skip-mounting-local-sources`` flag when running Breeze. In this case you will have sources
 embedded in the container and changes to these sources will not be persistent.
 
 
@@ -847,7 +847,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
    Manage mounting local files
   ****************************************************************************************************
 
-  -l, --skip-mounting-source-volume
+  -l, --skip-mounting-local-sources
           Skips mounting local volume with sources - you get exactly what is in the
           docker image rather than your current local sources of airflow.
 

diff --git a/breeze b/breeze
@@ -84,7 +84,7 @@ function setup_default_breeze_variables() {
     . "${MY_DIR}/breeze-complete"
 
     # Skips mounting local Airflow sources
-    SKIP_MOUNTING_LOCAL_SOURCES="false"
+    MOUNT_LOCAL_SOURCES="true"
 
     # Holds last subcommand used
     LAST_SUBCOMMAND=""
@@ -115,9 +115,6 @@ function setup_default_breeze_variables() {
     # Do not enable Kind Kubernetes cluster by default
     export ENABLE_KIND_CLUSTER="false"
 
-    # We use docker image caches by default to speed up the builds
-    export USE_PULLED_IMAGES_AS_CACHE=${USE_PULLED_IMAGES_AS_CACHE:="true"}
-
     # By default we do not push images. This can be overridden by -u flag.
     export PUSH_IMAGES=${PUSH_IMAGES:="false"}
 
@@ -129,15 +126,9 @@ function setup_default_breeze_variables() {
     export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:="current"}
 
     # Determine version of the Airflow from version.py
-    AIRFLOW_VERSION=$(cat airflow/version.py - << EOF | python
-print(version.replace("+",""))
-EOF
-    )
+    AIRFLOW_VERSION=$(grep version "airflow/version.py" | awk '{print $3}' | sed "s/['+]//g")
     export AIRFLOW_VERSION
 
-    # Verbosity in running ci scripts
-    export VERBOSE=${VERBOSE:="false"}
-
     # Whether to force build without checking if it is needed
     export FORCE_BUILD_IMAGES=${FORCE_BUILD_IMAGES:="false"}
 
@@ -283,7 +274,7 @@ EOF
 }
 
 function print_badge {
-    if [[ ! -f "${SUPPRESS_ASCIIART_FILE}" ]]; then
+    if [[ ! -f "${SUPPRESS_ASCIIART_FILE}" && ${COMMAND_TO_RUN} == "enter_breeze" ]]; then
         cat <<EOF
 
 
@@ -414,7 +405,7 @@ function prepare_command_files() {
 
     COMPOSE_FILE=${MAIN_DOCKER_COMPOSE_FILE}:${BACKEND_DOCKER_COMPOSE_FILE}
 
-    if [[ "${SKIP_MOUNTING_LOCAL_SOURCES}" != "true" ]]; then
+    if [[ "${MOUNT_LOCAL_SOURCES}" != "false" ]]; then
         COMPOSE_FILE=${COMPOSE_FILE}:${LOCAL_DOCKER_COMPOSE_FILE}
     fi
 
@@ -553,8 +544,8 @@ function parse_arguments() {
           echo
           shift 2 ;;
         -l|--skip-mounting-local-sources)
-          SKIP_MOUNTING_LOCAL_SOURCES="true"
-          echo "Skip mounting local sources: ${SKIP_MOUNTING_LOCAL_SOURCES}"
+          MOUNT_LOCAL_SOURCES="false"
+          echo "Mount local sources: ${MOUNT_LOCAL_SOURCES}"
           echo
           shift ;;
         -a|--install-airflow-version)
@@ -592,7 +583,7 @@ function parse_arguments() {
         -C|--force-clean-images)
           echo "Clean build of images without cache"
           echo
-          export USE_PULLED_IMAGES_AS_CACHE="false"
+          export DOCKER_CACHE="no-cache"
           export FORCE_BUILD_IMAGES="true"
           shift ;;
         -s|--kind-cluster-start)
@@ -619,8 +610,7 @@ function parse_arguments() {
         -L|--use-local-cache)
           echo "Use local cache to build images"
           echo
-          export USE_NO_CACHE="false"
-          export USE_PULLED_IMAGES_AS_CACHE="false"
+          export DOCKER_CACHE="local"
           shift ;;
         -P|--force-pull-images)
           echo "Force pulling images before build. Uses pulled images as cache."
@@ -1134,7 +1124,7 @@ $(print_star_line)
  Manage mounting local files
 $(print_star_line)
 
--l, --skip-mounting-source-volume
+-l, --skip-mounting-local-sources
         Skips mounting local volume with sources - you get exactly what is in the
         docker image rather than your current local sources of airflow.
 
@@ -1298,7 +1288,7 @@ function touch_local_files {
 }
 
 function print_cheatsheet() {
-    if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} ]]; then
+    if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} && ${COMMAND_TO_RUN} == "enter_breeze" ]]; then
         echo
         echo
         print_line
@@ -1353,25 +1343,27 @@ function print_cheatsheet() {
 }
 
 function print_setup_instructions {
-    # shellcheck disable=SC2034  # Unused variables left for comp_breeze usage
-    if ! typeset -f "_comp_breeze" > /dev/null; then
+    if [[ ${COMMAND_TO_RUN} == "enter_breeze" ]] ; then
+        # shellcheck disable=SC2034  # Unused variables left for comp_breeze usage
+        if ! typeset -f "_comp_breeze" > /dev/null; then
+            print_line
+            echo
+            echo "  You can setup autocomplete by running '${CMDNAME} setup-autocomplete'"
+            echo
+            echo
+        fi
+        print_line
+        echo
+        echo "  You can toggle ascii/cheatsheet by running:"
+        echo "      * ${CMDNAME} toggle-suppress-cheatsheet"
+        echo "      * ${CMDNAME} toggle-suppress-asciiart"
+        echo
         print_line
         echo
-        echo "  You can setup autocomplete by running '${CMDNAME} setup-autocomplete'"
+        echo
         echo
         echo
     fi
-    print_line
-    echo
-    echo "  You can toggle ascii/cheatsheet by running:"
-    echo "      * ${CMDNAME} toggle-suppress-cheatsheet"
-    echo "      * ${CMDNAME} toggle-suppress-asciiart"
-    echo
-    print_line
-    echo
-    echo
-    echo
-    echo
 }
 
 function make_sure_precommit_is_installed {
@@ -1415,6 +1407,7 @@ function run_static_checks {
 }
 
 function run_build_command {
+    prepare_build
     case "${COMMAND_TO_RUN}" in
         enter_breeze|build_docs|run_tests|run_docker_compose|run_in_bash)
             rebuild_ci_image_if_needed

diff --git a/confirm b/confirm
@@ -31,21 +31,21 @@ if [[ "${FORCE_ANSWER_TO_QUESTIONS:=""}" != "" ]]; then
     esac
 else
     echo
-    echo "Please confirm ${1} images. Are you sure? [y/N/q]"
+    echo "Please confirm ${1}. Are you sure? [y/N/q]"
     read -r RESPONSE
 fi
 
 case "${RESPONSE}" in
     [yY][eE][sS]|[yY])
-        echo "The answer is 'yes'. Attempting to ${1} images. This can take some time !"
+        echo "The answer is 'yes'. ${1}. This can take some time !"
         exit 0
         ;;
     [qQ][uU][iI][tT]|[qQ])
         echo "The answer is 'quit'. Quitting now!"
         exit 2
         ;;
     *)
-        echo "The answer is 'no'. Skipping attempt to ${1} images."
+        echo "The answer is 'no'. Skipping ${1}."
         exit 1
         ;;
 esac