From 963c9713167eb3ba228fed38dac8f1c6a45f7a36 Mon Sep 17 00:00:00 2001
From: Jarek Potiuk <jarek.potiuk@polidea.com>
Date: Sat, 7 Mar 2020 18:12:28 +0100
Subject: [PATCH] [AIRFLOW-7002] Get rid of yaml "parser" in bash (#7646)

Mounts in docker compose file is now generated from an env variable
---
 .pre-commit-config.yaml                   |   7 +
 scripts/ci/_utils.sh                      |  92 ++++++-------
 scripts/ci/docker-compose/local.yml       |   4 +-
 scripts/ci/pre_commit_local_yml_mounts.sh |  51 ++++++++
 tests/bats/test_local_mounts.bats         |  91 +++++++++++++
 tests/bats/test_yaml_parser.bats          | 149 ----------------------
 6 files changed, 194 insertions(+), 200 deletions(-)
 create mode 100755 scripts/ci/pre_commit_local_yml_mounts.sh
 create mode 100644 tests/bats/test_local_mounts.bats
 delete mode 100644 tests/bats/test_yaml_parser.bats

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4d58e258ec89be..206ee159262e8f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -202,6 +202,13 @@ repos:
         files: ^BREEZE.rst$|^breeze$|^breeze-complete$
         pass_filenames: false
         require_serial: true
+      - id: update-local-yml-file
+        name: Update mounts in the local yml file
+        entry: "./scripts/ci/pre_commit_local_yml_mounts.sh"
+        language: system
+        files: ^scripts/ci/_utils.sh$|s^scripts/ci/docker_compose/local.yml"
+        pass_filenames: false
+        require_serial: true
       - id: update-setup-cfg-file
         name: Update setup.cfg file with all licenses
         entry: "./scripts/ci/pre_commit_setup_cfg_file.sh"
diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh
index bd5ef11eec7525..c2c88fd680d1d1 100644
--- a/scripts/ci/_utils.sh
+++ b/scripts/ci/_utils.sh
@@ -114,7 +114,7 @@ function initialize_breeze_environment {
 
         while IFS= read -r LINE; do
             EXTRA_DOCKER_FLAGS+=( "${LINE}")
-        done < <(convert_docker_mounts_to_docker_params)
+        done < <(convert_local_mounts_to_docker_params)
     else
         print_info
         print_info "Skip mounting host volumes to Docker"
@@ -148,60 +148,52 @@ function print_info() {
     fi
 }
 
-# shellcheck disable=SC1087
-# Simple (?) no-dependency needed Yaml PARSER
-# From https://stackoverflow.com/questions/5014632/how-can-i-parse-a-yaml-file-from-a-linux-shell-script
-function parse_yaml {
-    if [[ -z $1 ]]; then
-        echo "Please provide yaml filename as first parameter."
-        exit 1
-    fi
-    local prefix=$2
-    local s='[[:space:]]*' w='[a-zA-Z0-9_-]*' FS
-    FS=$(echo @|tr @ '\034')
-    sed -ne "s|,$s\]$s\$|]|" \
-         -e ":1;s|^\($s\)\($w\)$s:$s\[$s\(.*\)$s,$s\(.*\)$s\]|\1\2: [\3]\n\1  - \4|;t1" \
-         -e "s|^\($s\)\($w\)$s:$s\[$s\(.*\)$s\]|\1\2:\n\1  - \3|;p" "${1}" | \
-    sed -ne "s|,$s}$s\$|}|" \
-         -e ":1;s|^\($s\)-$s{$s\(.*\)$s,$s\($w\)$s:$s\(.*\)$s}|\1- {\2}\n\1  \3: \4|;t1" \
-         -e    "s|^\($s\)-$s{$s\(.*\)$s}|\1-\n\1  \2|;p" | \
-    sed -ne "s|^\($s\):|\1|" \
-         -e "s|^\($s\)-$s[\"']\(.*\)[\"']$s\$|\1$FS$FS\2|p" \
-         -e "s|^\($s\)-$s\(.*\)$s\$|\1$FS$FS\2|p" \
-         -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$FS\2$FS\3|p" \
-         -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$FS\2$FS\3|p" | \
-    awk -F"${FS}" '{
-       indent = length($1)/2;
-       vname[indent] = $2;
-       for (i in vname) {if (i > indent) {delete vname[i]; idx[i]=0}}
-       if(length($2)== 0){  vname[indent]= ++idx[indent] };
-       if (length($3) > 0) {
-          vn=""; for (i=0; i<indent; i++) { vn=(vn)(vname[i])("_")}
-          printf("%s%s%s=\"%s\"\n", "'"${prefix}"'",vn, vname[indent], $3);
-       }
-    }'
-}
+LOCAL_MOUNTS="
+.bash_aliases /root/
+.bash_history /root/
+.coveragerc /opt/airflow/
+.dockerignore /opt/airflow/
+.flake8 /opt/airflow/
+.github /opt/airflow/
+.inputrc /root/
+.kube /root/
+.rat-excludes /opt/airflow/
+CHANGELOG.txt /opt/airflow/
+Dockerfile /opt/airflow/
+LICENSE /opt/airflow/
+MANIFEST.in /opt/airflow/
+NOTICE /opt/airflow/
+airflow /opt/airflow/
+common /opt/airflow/
+dags /opt/airflow/
+dev /opt/airflow/
+docs /opt/airflow/
+files /
+dist /
+hooks /opt/airflow/
+logs /root/airflow/
+pylintrc /opt/airflow/
+pytest.ini /opt/airflow/
+scripts /opt/airflow/
+scripts/ci/in_container/entrypoint_ci.sh /
+setup.cfg /opt/airflow/
+setup.py /opt/airflow/
+tests /opt/airflow/
+tmp /opt/airflow/
+"
 
 # parse docker-compose-local.yaml file to convert volumes entries
 # from airflow-testing section to "-v" "volume mapping" series of options
-function convert_docker_mounts_to_docker_params() {
-    ESCAPED_AIRFLOW_SOURCES=$(echo "${AIRFLOW_SOURCES}" | sed -e 's/[\/&]/\\&/g')
-    ESCAPED_HOME=$(echo "${HOME}" | sed -e 's/[\/&]/\\&/g')
-    # shellcheck disable=2046
-    while IFS= read -r LINE; do
-        echo "-v"
-        echo "${LINE}"
-    done < <(parse_yaml scripts/ci/docker-compose/local.yml COMPOSE_ | \
-            grep "COMPOSE_services_airflow-testing_volumes_" | \
-            sed "s/..\/..\/../${ESCAPED_AIRFLOW_SOURCES}/" | \
-            sed "s/\${HOME}/${ESCAPED_HOME}/" | \
-            sed "s/COMPOSE_services_airflow-testing_volumes_//" | \
-            sort -t "=" -k 1 -n | \
-            cut -d "=" -f 2- | \
-            sed -e 's/^"//' -e 's/"$//')
+function convert_local_mounts_to_docker_params() {
+    echo "${LOCAL_MOUNTS}" |sed '/^$/d' | awk -v AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \
+    '
+    function basename(file) {
+        sub(".*/", "", file)
+        return file
+    }
+    { print "-v"; print AIRFLOW_SOURCES "/" $1 ":" $2 basename($1) ":cached" }'
 }
 
-
 function sanitize_file() {
     if [[ -d "${1}" ]]; then
         rm -rf "${1}"
diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml
index bd7872f5237a72..603782e05cace1 100644
--- a/scripts/ci/docker-compose/local.yml
+++ b/scripts/ci/docker-compose/local.yml
@@ -24,6 +24,7 @@ services:
     # or those that might be useful to see in the host as output of the
     # tests (such as logs)
     volumes:
+      # START automatically generated volumes from LOCAL_MOUNTS in _utils.sh
       - ../../../.bash_aliases:/root/.bash_aliases:cached
       - ../../../.bash_history:/root/.bash_history:cached
       - ../../../.coveragerc:/opt/airflow/.coveragerc:cached
@@ -33,7 +34,7 @@ services:
       - ../../../.inputrc:/root/.inputrc:cached
       - ../../../.kube:/root/.kube:cached
       - ../../../.rat-excludes:/opt/airflow/.rat-excludes:cached
-      - ../../../CHANGELOG.txt:/opt/airflow/CHANGELOG:cached
+      - ../../../CHANGELOG.txt:/opt/airflow/CHANGELOG.txt:cached
       - ../../../Dockerfile:/opt/airflow/Dockerfile:cached
       - ../../../LICENSE:/opt/airflow/LICENSE:cached
       - ../../../MANIFEST.in:/opt/airflow/MANIFEST.in:cached
@@ -55,6 +56,7 @@ services:
       - ../../../setup.py:/opt/airflow/setup.py:cached
       - ../../../tests:/opt/airflow/tests:cached
       - ../../../tmp:/opt/airflow/tmp:cached
+      # END automatically generated volumes from LOCAL_MOUNTS in _utils.sh
     environment:
       - HOST_USER_ID
       - HOST_GROUP_ID
diff --git a/scripts/ci/pre_commit_local_yml_mounts.sh b/scripts/ci/pre_commit_local_yml_mounts.sh
new file mode 100755
index 00000000000000..a61418416d62df
--- /dev/null
+++ b/scripts/ci/pre_commit_local_yml_mounts.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# shellcheck source=scripts/ci/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
+
+TMP_FILE=$(mktemp)
+TMP_OUTPUT=$(mktemp)
+
+LOCAL_YML_FILE="${MY_DIR}/docker-compose/local.yml"
+
+LEAD='      # START automatically generated volumes from LOCAL_MOUNTS in _utils.sh'
+TAIL='      # END automatically generated volumes from LOCAL_MOUNTS in _utils.sh'
+
+echo "${LOCAL_MOUNTS}" |sed '/^$/d' | \
+    awk '
+    function basename(file) {
+        sub(".*/", "", file)
+        return file
+    }
+    { print "      - ../../../" $1 ":" $2 basename($1) ":cached"}
+    ' > "${TMP_FILE}"
+
+
+BEGIN_GEN=$(grep -n "${LEAD}" <"${LOCAL_YML_FILE}" | sed 's/\(.*\):.*/\1/g')
+END_GEN=$(grep -n "${TAIL}" <"${LOCAL_YML_FILE}" | sed 's/\(.*\):.*/\1/g')
+cat <(head -n "${BEGIN_GEN}" "${LOCAL_YML_FILE}") \
+    "${TMP_FILE}" \
+    <(tail -n +"${END_GEN}" "${LOCAL_YML_FILE}") \
+    >"${TMP_OUTPUT}"
+
+mv "${TMP_OUTPUT}" "${LOCAL_YML_FILE}"
diff --git a/tests/bats/test_local_mounts.bats b/tests/bats/test_local_mounts.bats
new file mode 100644
index 00000000000000..178cacd6ee98e6
--- /dev/null
+++ b/tests/bats/test_local_mounts.bats
@@ -0,0 +1,91 @@
+#!/usr/bin/env bats
+
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+@test "convert volume list to docker params" {
+  load bats_utils
+
+  initialize_breeze_environment
+
+  run convert_local_mounts_to_docker_params
+  diff <(echo "${output}") - << EOF
+-v
+${AIRFLOW_SOURCES}/.bash_aliases:/root/.bash_aliases:cached
+-v
+${AIRFLOW_SOURCES}/.bash_history:/root/.bash_history:cached
+-v
+${AIRFLOW_SOURCES}/.coveragerc:/opt/airflow/.coveragerc:cached
+-v
+${AIRFLOW_SOURCES}/.dockerignore:/opt/airflow/.dockerignore:cached
+-v
+${AIRFLOW_SOURCES}/.flake8:/opt/airflow/.flake8:cached
+-v
+${AIRFLOW_SOURCES}/.github:/opt/airflow/.github:cached
+-v
+${AIRFLOW_SOURCES}/.inputrc:/root/.inputrc:cached
+-v
+${AIRFLOW_SOURCES}/.kube:/root/.kube:cached
+-v
+${AIRFLOW_SOURCES}/.rat-excludes:/opt/airflow/.rat-excludes:cached
+-v
+${AIRFLOW_SOURCES}/CHANGELOG.txt:/opt/airflow/CHANGELOG.txt:cached
+-v
+${AIRFLOW_SOURCES}/Dockerfile:/opt/airflow/Dockerfile:cached
+-v
+${AIRFLOW_SOURCES}/LICENSE:/opt/airflow/LICENSE:cached
+-v
+${AIRFLOW_SOURCES}/MANIFEST.in:/opt/airflow/MANIFEST.in:cached
+-v
+${AIRFLOW_SOURCES}/NOTICE:/opt/airflow/NOTICE:cached
+-v
+${AIRFLOW_SOURCES}/airflow:/opt/airflow/airflow:cached
+-v
+${AIRFLOW_SOURCES}/common:/opt/airflow/common:cached
+-v
+${AIRFLOW_SOURCES}/dags:/opt/airflow/dags:cached
+-v
+${AIRFLOW_SOURCES}/dev:/opt/airflow/dev:cached
+-v
+${AIRFLOW_SOURCES}/docs:/opt/airflow/docs:cached
+-v
+${AIRFLOW_SOURCES}/files:/files:cached
+-v
+${AIRFLOW_SOURCES}/dist:/dist:cached
+-v
+${AIRFLOW_SOURCES}/hooks:/opt/airflow/hooks:cached
+-v
+${AIRFLOW_SOURCES}/logs:/root/airflow/logs:cached
+-v
+${AIRFLOW_SOURCES}/pylintrc:/opt/airflow/pylintrc:cached
+-v
+${AIRFLOW_SOURCES}/pytest.ini:/opt/airflow/pytest.ini:cached
+-v
+${AIRFLOW_SOURCES}/scripts:/opt/airflow/scripts:cached
+-v
+${AIRFLOW_SOURCES}/scripts/ci/in_container/entrypoint_ci.sh:/entrypoint_ci.sh:cached
+-v
+${AIRFLOW_SOURCES}/setup.cfg:/opt/airflow/setup.cfg:cached
+-v
+${AIRFLOW_SOURCES}/setup.py:/opt/airflow/setup.py:cached
+-v
+${AIRFLOW_SOURCES}/tests:/opt/airflow/tests:cached
+-v
+${AIRFLOW_SOURCES}/tmp:/opt/airflow/tmp:cached
+EOF
+}
diff --git a/tests/bats/test_yaml_parser.bats b/tests/bats/test_yaml_parser.bats
deleted file mode 100644
index d633bb5eb54c3e..00000000000000
--- a/tests/bats/test_yaml_parser.bats
+++ /dev/null
@@ -1,149 +0,0 @@
-#!/usr/bin/env bats
-
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-@test "yaml parsing missing file" {
-  load bats_utils
-
-  initialize_breeze_environment
-
-  run parse_yaml
-  [ "${status}" == 1 ]
-  [ "${output}" == "Please provide yaml filename as first parameter." ]
-}
-
-@test "yaml parsing existing file" {
-  load bats_utils
-
-  initialize_breeze_environment
-
-  run parse_yaml "scripts/ci/docker-compose/local.yml"
-  diff <(echo "${output}") - << 'EOF'
-1="--"
-version="2.2"
-services_airflow-testing_volumes_1="../../../.bash_aliases:/root/.bash_aliases:cached"
-services_airflow-testing_volumes_2="../../../.bash_history:/root/.bash_history:cached"
-services_airflow-testing_volumes_3="../../../.coveragerc:/opt/airflow/.coveragerc:cached"
-services_airflow-testing_volumes_4="../../../.dockerignore:/opt/airflow/.dockerignore:cached"
-services_airflow-testing_volumes_5="../../../.flake8:/opt/airflow/.flake8:cached"
-services_airflow-testing_volumes_6="../../../.github:/opt/airflow/.github:cached"
-services_airflow-testing_volumes_7="../../../.inputrc:/root/.inputrc:cached"
-services_airflow-testing_volumes_8="../../../.kube:/root/.kube:cached"
-services_airflow-testing_volumes_9="../../../.rat-excludes:/opt/airflow/.rat-excludes:cached"
-services_airflow-testing_volumes_10="../../../CHANGELOG.txt:/opt/airflow/CHANGELOG:cached"
-services_airflow-testing_volumes_11="../../../Dockerfile:/opt/airflow/Dockerfile:cached"
-services_airflow-testing_volumes_12="../../../LICENSE:/opt/airflow/LICENSE:cached"
-services_airflow-testing_volumes_13="../../../MANIFEST.in:/opt/airflow/MANIFEST.in:cached"
-services_airflow-testing_volumes_14="../../../NOTICE:/opt/airflow/NOTICE:cached"
-services_airflow-testing_volumes_15="../../../airflow:/opt/airflow/airflow:cached"
-services_airflow-testing_volumes_16="../../../common:/opt/airflow/common:cached"
-services_airflow-testing_volumes_17="../../../dags:/opt/airflow/dags:cached"
-services_airflow-testing_volumes_18="../../../dev:/opt/airflow/dev:cached"
-services_airflow-testing_volumes_19="../../../docs:/opt/airflow/docs:cached"
-services_airflow-testing_volumes_20="../../../files:/files:cached"
-services_airflow-testing_volumes_21="../../../dist:/dist:cached"
-services_airflow-testing_volumes_22="../../../hooks:/opt/airflow/hooks:cached"
-services_airflow-testing_volumes_23="../../../logs:/root/airflow/logs:cached"
-services_airflow-testing_volumes_24="../../../pylintrc:/opt/airflow/pylintrc:cached"
-services_airflow-testing_volumes_25="../../../pytest.ini:/opt/airflow/pytest.ini:cached"
-services_airflow-testing_volumes_26="../../../scripts:/opt/airflow/scripts:cached"
-services_airflow-testing_volumes_27="../../../scripts/ci/in_container/entrypoint_ci.sh:/entrypoint_ci.sh:cached"
-services_airflow-testing_volumes_28="../../../setup.cfg:/opt/airflow/setup.cfg:cached"
-services_airflow-testing_volumes_29="../../../setup.py:/opt/airflow/setup.py:cached"
-services_airflow-testing_volumes_30="../../../tests:/opt/airflow/tests:cached"
-services_airflow-testing_volumes_31="../../../tmp:/opt/airflow/tmp:cached"
-services_airflow-testing_environment_1="HOST_USER_ID"
-services_airflow-testing_environment_2="HOST_GROUP_ID"
-services_airflow-testing_environment_3="PYTHONDONTWRITEBYTECODE"
-services_airflow-testing_ports_1="${WEBSERVER_HOST_PORT}:8080"
-EOF
-}
-
-
-@test "convert yaml docker file to docker params" {
-  load bats_utils
-
-  initialize_breeze_environment
-
-  run convert_docker_mounts_to_docker_params
-  diff <(echo "${output}") - << EOF
--v
-${AIRFLOW_SOURCES}/.bash_aliases:/root/.bash_aliases:cached
--v
-${AIRFLOW_SOURCES}/.bash_history:/root/.bash_history:cached
--v
-${AIRFLOW_SOURCES}/.coveragerc:/opt/airflow/.coveragerc:cached
--v
-${AIRFLOW_SOURCES}/.dockerignore:/opt/airflow/.dockerignore:cached
--v
-${AIRFLOW_SOURCES}/.flake8:/opt/airflow/.flake8:cached
--v
-${AIRFLOW_SOURCES}/.github:/opt/airflow/.github:cached
--v
-${AIRFLOW_SOURCES}/.inputrc:/root/.inputrc:cached
--v
-${AIRFLOW_SOURCES}/.kube:/root/.kube:cached
--v
-${AIRFLOW_SOURCES}/.rat-excludes:/opt/airflow/.rat-excludes:cached
--v
-${AIRFLOW_SOURCES}/CHANGELOG.txt:/opt/airflow/CHANGELOG:cached
--v
-${AIRFLOW_SOURCES}/Dockerfile:/opt/airflow/Dockerfile:cached
--v
-${AIRFLOW_SOURCES}/LICENSE:/opt/airflow/LICENSE:cached
--v
-${AIRFLOW_SOURCES}/MANIFEST.in:/opt/airflow/MANIFEST.in:cached
--v
-${AIRFLOW_SOURCES}/NOTICE:/opt/airflow/NOTICE:cached
--v
-${AIRFLOW_SOURCES}/airflow:/opt/airflow/airflow:cached
--v
-${AIRFLOW_SOURCES}/common:/opt/airflow/common:cached
--v
-${AIRFLOW_SOURCES}/dags:/opt/airflow/dags:cached
--v
-${AIRFLOW_SOURCES}/dev:/opt/airflow/dev:cached
--v
-${AIRFLOW_SOURCES}/docs:/opt/airflow/docs:cached
--v
-${AIRFLOW_SOURCES}/files:/files:cached
--v
-${AIRFLOW_SOURCES}/dist:/dist:cached
--v
-${AIRFLOW_SOURCES}/hooks:/opt/airflow/hooks:cached
--v
-${AIRFLOW_SOURCES}/logs:/root/airflow/logs:cached
--v
-${AIRFLOW_SOURCES}/pylintrc:/opt/airflow/pylintrc:cached
--v
-${AIRFLOW_SOURCES}/pytest.ini:/opt/airflow/pytest.ini:cached
--v
-${AIRFLOW_SOURCES}/scripts:/opt/airflow/scripts:cached
--v
-${AIRFLOW_SOURCES}/scripts/ci/in_container/entrypoint_ci.sh:/entrypoint_ci.sh:cached
--v
-${AIRFLOW_SOURCES}/setup.cfg:/opt/airflow/setup.cfg:cached
--v
-${AIRFLOW_SOURCES}/setup.py:/opt/airflow/setup.py:cached
--v
-${AIRFLOW_SOURCES}/tests:/opt/airflow/tests:cached
--v
-${AIRFLOW_SOURCES}/tmp:/opt/airflow/tmp:cached
-EOF
-}