Group CI scripts in subdirectories (#9653)

Reviewed the scripts and removed some of the old unused ones. (cherry picked from commit faec41e)
apache · Jul 22, 2020 · 07d0c84 · 07d0c84
1 parent dff4443
commit 07d0c84
Show file tree

Hide file tree

Showing 59 changed files with 355 additions and 494 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -60,15 +60,15 @@ jobs:
           path: ~/.cache/pre-commit
           key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Static checks"
         if: success()
         run: |
           python -m pip install pre-commit \
               --constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
-          ./scripts/ci/ci_run_static_checks.sh
+          ./scripts/ci/static_checks/ci_run_static_checks.sh
 
   docs:
     timeout-minutes: 60
@@ -82,9 +82,9 @@ jobs:
         with:
           python-version: '3.6'
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Build docs"
-        run: ./scripts/ci/ci_docs.sh
+        run: ./scripts/ci/docs/ci_docs.sh
 
   build-prod-image:
     timeout-minutes: 60
@@ -99,7 +99,7 @@ jobs:
     steps:
       - uses: actions/checkout@master
       - name: "Build PROD image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
 
   trigger-tests:
     timeout-minutes: 10
@@ -112,7 +112,7 @@ jobs:
       - name: "Get count of changed python files"
         run: |
           set +e
-          ./scripts/ci/ci_count_changed_files.sh ${GITHUB_SHA} \
+          ./scripts/ci/tools/ci_count_changed_files.sh ${GITHUB_SHA} \
               '^airflow|.github/workflows/|^Dockerfile|^scripts|^chart|^setup.py|^requirements|^tests|^kubernetes_tests'
           echo "::set-output name=count::$?"
         id: trigger-tests
@@ -155,15 +155,15 @@ jobs:
         with:
           python-version: '3.6'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - uses: engineerd/[email protected]
         name: Setup Kind Cluster
         with:
           version: "${{ matrix.kind-version }}"
           name: airflow-python-${{matrix.python-version}}-${{matrix.kubernetes-version}}
           config: "scripts/ci/kubernetes/kind-cluster-conf.yaml"
       - name: "Deploy app to cluster"
-        run: ./scripts/ci/ci_deploy_app_to_kubernetes.sh
+        run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
       - name: Cache virtualenv for kubernetes testing
         uses: actions/cache@v2
         env:
@@ -173,7 +173,7 @@ jobs:
           key: "${{ env.cache-name }}-${{ github.job }}-\
 ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') }}"
       - name: "Tests"
-        run: ./scripts/ci/ci_run_kubernetes_tests.sh
+        run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
       - uses: actions/upload-artifact@v2
         name: Upload KinD logs
         # Always run this, even if one of th previous steps failed.
@@ -208,11 +208,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
         with:
           python-version: '3.6'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Tests"
-        run: ./scripts/ci/ci_run_airflow_testing.sh
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
 
   tests-mysql:
     timeout-minutes: 80
@@ -240,11 +240,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
         with:
           python-version: '3.x'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Tests"
-        run: ./scripts/ci/ci_run_airflow_testing.sh
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
 
   tests-sqlite:
     timeout-minutes: 80
@@ -270,11 +270,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
         with:
           python-version: '3.x'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Tests"
-        run: ./scripts/ci/ci_run_airflow_testing.sh
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
 
   tests-quarantined:
     timeout-minutes: 80
@@ -303,11 +303,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
         with:
           python-version: '3.x'
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Tests"
-        run: ./scripts/ci/ci_run_airflow_testing.sh
+        run: ./scripts/ci/testing/ci_run_airflow_testing.sh
 
   helm-tests:
     timeout-minutes: 5
@@ -337,11 +337,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
       - uses: actions/checkout@master
       - uses: actions/setup-python@v1
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Generate requirements"
-        run: ./scripts/ci/ci_generate_requirements.sh
+        run: ./scripts/ci/requirements/ci_generate_requirements.sh
 
   push-prod-images-to-github-cache:
     timeout-minutes: 80
@@ -364,11 +364,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     steps:
       - uses: actions/checkout@master
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build PROD images ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
       - name: "Push PROD images ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_push_production_images.sh
+        run: ./scripts/ci/images/ci_push_production_images.sh
 
   push-ci-images-to-github-cache:
     timeout-minutes: 40
@@ -395,8 +395,8 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
     steps:
       - uses: actions/checkout@master
       - name: "Free space"
-        run: ./scripts/ci/ci_free_space_on_ci.sh
+        run: ./scripts/ci/tools/ci_free_space_on_ci.sh
       - name: "Build CI image"
-        run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
+        run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
       - name: "Push CI image ${{ matrix.python-version }}"
-        run: ./scripts/ci/ci_push_ci_image.sh
+        run: ./scripts/ci/images/ci_push_ci_image.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -198,7 +198,7 @@ repos:
       - id: lint-dockerfile
         name: Lint dockerfile
         language: system
-        entry: "./scripts/ci/pre_commit_lint_dockerfile.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh"
         files: ^Dockerfile.*$
         pass_filenames: true
       - id: setup-order
@@ -207,25 +207,25 @@ repos:
         files: ^setup.py$
         pass_filenames: false
         require_serial: true
-        entry: tests/test_order_setup.py
+        entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
       - id: update-breeze-file
         name: Update output of breeze command in BREEZE.rst
-        entry: "./scripts/ci/pre_commit_breeze_cmd_line.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh"
         language: system
         files: ^BREEZE.rst$|^breeze$|^breeze-complete$
         pass_filenames: false
         require_serial: true
       - id: update-local-yml-file
         name: Update mounts in the local yml file
-        entry: "./scripts/ci/pre_commit_local_yml_mounts.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh"
         language: system
         files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml"
         pass_filenames: false
         require_serial: true
       - id: update-extras
         name: Update extras in documentation
-        entry: "./scripts/ci/pre_commit_update_extras.sh"
-        language: system
+        entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py
+        language: python
         files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$
         pass_filenames: false
         require_serial: true
@@ -265,14 +265,14 @@ repos:
                 ^\sdef\s*\S*\(.*\):\s*\-\>\s*\S*.*  # Matches -> return value syntax from Python3
             )$
         files: \.py$
-        exclude: ^dev/
+        exclude: ^dev|^scripts
         pass_filenames: true
       - id: python2-compile
         name: Compile code using python2
         language: system
         entry: python2.7 -m py_compile
         files: \.py$
-        exclude: ^dev/
+        exclude: ^dev|^scripts
         pass_filenames: true
         require_serial: true
       - id: pydevd
@@ -283,7 +283,7 @@ repos:
         pass_filenames: true
       - id: check-integrations
         name: Check if integration list is aligned
-        entry: ./scripts/ci/pre_commit_check_integrations.sh
+        entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
         language: system
         pass_filenames: false
         files: ^common/_common_values.sh$|^breeze-complete$
@@ -295,42 +295,42 @@ repos:
         pass_filenames: true
       - id: build
         name: Check if image build is needed
-        entry: ./scripts/ci/pre_commit_ci_build.sh 3.5 false
+        entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false
         language: system
         always_run: true
         pass_filenames: false
       - id: check-apache-license
         name: Check if licenses are OK for Apache
-        entry: "./scripts/ci/pre_commit_check_license.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh"
         language: system
         files: ^.*LICENSE.*$|^.*LICENCE.*$
         pass_filenames: false
         require_serial: true
       - id: airflow-config-yaml
         name: Checks for consistency between config.yml and default_config.cfg
         language: python
+        entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
         files: "^airflow/config_templates/config.yml$|^airflow/config_templates/default_airflow.cfg$"
         pass_filenames: false
-        require_serial: false
-        entry: scripts/ci/pre_commit_yaml_to_cfg.py
+        require_serial: true
         additional_dependencies: ['pyyaml']
       - id: mypy
         name: Run mypy
         language: system
-        entry: "./scripts/ci/pre_commit_mypy.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
         files: \.py$
         exclude: ^dev
         require_serial: true
       - id: flake8
         name: Run flake8
         language: system
-        entry: "./scripts/ci/pre_commit_flake8.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh"
         files: \.py$
-        exclude: ^dev/
+        exclude: ^dev
         pass_filenames: true
       - id: bat-tests
         name: Run BATS bash tests for changed bash files
         language: system
-        entry: "./scripts/ci/pre_commit_bat_tests.sh"
+        entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh"
         files: ^breeze$|^breeze-complete$|\.sh$|\.bash$
         pass_filenames: false
diff --git a/.rat-excludes b/.rat-excludes
@@ -86,3 +86,4 @@ input_notebook.ipynb
 
 # .git might be a file in case of worktree
 .git
+tmp
diff --git a/BREEZE.rst b/BREEZE.rst
@@ -920,7 +920,7 @@ by the root user, you can fix the ownership of those files by running this scrip
 
 .. code-block::
 
-  ./scripts/ci/ci_fix_ownership.sh
+  ./scripts/ci/tools/ci_fix_ownership.sh
 
 Mounting Local Sources to Breeze
 --------------------------------

diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
@@ -265,13 +265,12 @@ Running Static Code Checks via Scripts from the Host
 ....................................................
 
 You can trigger the static checks from the host environment, without entering the Docker container. To do
-this, run the following scripts (the same is done in the CI builds):
+this, run the following scripts:
 
-* `<scripts/ci/ci_check_license.sh>`_ - checks the licenses.
-* `<scripts/ci/ci_docs.sh>`_ - checks that documentation can be built without warnings.
-* `<scripts/ci/ci_flake8.sh>`_ - runs Flake8 source code style enforcement tool.
-* `<scripts/ci/ci_lint_dockerfile.sh>`_ - runs lint checker for the dockerfiles.
-* `<scripts/ci/ci_mypy.sh>`_ - runs a check for mypy type annotation consistency.
+* `<scripts/ci/static_checks/ci_check_license.sh>`_ - checks the licenses.
+* `<scripts/ci/static_checks/ci_flake8.sh>`_ - runs Flake8 source code style enforcement tool.
+* `<scripts/ci/static_checks/ci_lint_dockerfile.sh>`_ - runs lint checker for the dockerfiles.
+* `<scripts/ci/static_checks/ci_mypy.sh>`_ - runs a check for mypy type annotation consistency.
 
 The scripts may ask you to rebuild the images, if needed.
 
@@ -314,8 +313,8 @@ On the host:
 
 .. code-block::
 
-  ./scripts/ci/ci_mypy.sh ./airflow/example_dags/
+  ./scripts/ci/static_checks/ci_mypy.sh ./airflow/example_dags/
 
 .. code-block::
 
-  ./scripts/ci/ci_mypy.sh ./airflow/example_dags/test_utils.py
+  ./scripts/ci/static_checks/ci_mypy.sh ./airflow/example_dags/test_utils.py
diff --git a/TESTING.rst b/TESTING.rst
@@ -378,10 +378,10 @@ to run the tests manually one by one.
 
     Running kubernetes tests
 
-      ./scripts/ci/ci_run_kubernetes_tests.sh                      - runs all kubernetes tests
-      ./scripts/ci/ci_run_kubernetes_tests.sh TEST [TEST ...]      - runs selected kubernetes tests (from kubernetes_tests folder)
-      ./scripts/ci/ci_run_kubernetes_tests.sh [-i|--interactive]   - Activates virtual environment ready to run tests and drops you in
-      ./scripts/ci/ci_run_kubernetes_tests.sh [--help]             - Prints this help message
+      ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh                      - runs all kubernetes tests
+      ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh TEST [TEST ...]      - runs selected kubernetes tests (from kubernetes_tests folder)
+      ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [-i|--interactive]   - Activates virtual environment ready to run tests and drops you in
+      ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [--help]             - Prints this help message
 
 
 You can also run the same tests command with Breeze, using ``kind-cluster test`` command (to run all

diff --git a/breeze b/breeze
@@ -62,8 +62,8 @@ function setup_default_breeze_variables() {
 
     # load all the common functions here - those are the functions that are shared between Breeze
     # and CI scripts (CI scripts do not use Breeze as execution environment)
-    # shellcheck source=scripts/ci/_all_libs.sh
-    . "${SCRIPTS_CI_DIR}/_all_libs.sh"
+    # shellcheck source=scripts/ci/libraries/_all_libs.sh
+    . "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
 
     # We have different versions of images depending on the python version used. We keep up with the
     # Latest patch-level changes in Python (this is done automatically during CI builds) so we have

diff --git a/docs/start_doc_server.sh b/docs/start_doc_server.sh
@@ -16,8 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-(cd "${MY_DIR}"/_build/html || exit;
+DOCS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+(cd "${DOCS_DIR}"/_build/html || exit;
 # The below command works on both Python 2 and Python 3
 python -m http.server 8000 && python -m SimpleHTTPServer 8000
 )
diff --git a/hooks/build b/hooks/build
@@ -20,9 +20,9 @@
 # on Travis CI to potentially rebuild (and refresh layers that
 # are not cached) Docker images that are used to run CI jobs
 
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+_HOOK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 # Dockerhub builds are run inside Docker container
 export SKIP_IN_CONTAINER_CHECK="true"
 
-exec "${MY_DIR}/../scripts/ci/ci_build_dockerhub.sh"
+exec "${_HOOK_DIR}/../scripts/ci/images/ci_build_dockerhub.sh"
diff --git a/hooks/push b/hooks/push
@@ -20,11 +20,6 @@
 # and it is difficult to pass list of the built images from the build to push phase
 set -euo pipefail
 
-MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-echo "My dir: ${MY_DIR}"
-
-
 echo
 echo "Skip pushing the image. All images were built and pushed in the build hook already!"
 echo