diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 134bc1fa0a7165..a849d58167009b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,15 +60,15 @@ jobs: path: ~/.cache/pre-commit key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }} - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Static checks" if: success() run: | python -m pip install pre-commit \ --constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt - ./scripts/ci/ci_run_static_checks.sh + ./scripts/ci/static_checks/ci_run_static_checks.sh docs: timeout-minutes: 60 @@ -82,9 +82,9 @@ jobs: with: python-version: '3.6' - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Build docs" - run: ./scripts/ci/ci_docs.sh + run: ./scripts/ci/docs/ci_docs.sh build-prod-image: timeout-minutes: 60 @@ -99,7 +99,7 @@ jobs: steps: - uses: actions/checkout@master - name: "Build PROD image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh trigger-tests: timeout-minutes: 10 @@ -112,7 +112,7 @@ jobs: - name: "Get count of changed python files" run: | set +e - ./scripts/ci/ci_count_changed_files.sh ${GITHUB_SHA} \ + ./scripts/ci/tools/ci_count_changed_files.sh ${GITHUB_SHA} \ '^airflow|.github/workflows/|^Dockerfile|^scripts|^chart|^setup.py|^requirements|^tests|^kubernetes_tests' echo "::set-output name=count::$?" id: trigger-tests @@ -155,7 +155,7 @@ jobs: with: python-version: '3.6' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - uses: engineerd/setup-kind@v0.4.0 name: Setup Kind Cluster with: @@ -163,7 +163,7 @@ jobs: name: airflow-python-${{matrix.python-version}}-${{matrix.kubernetes-version}} config: "scripts/ci/kubernetes/kind-cluster-conf.yaml" - name: "Deploy app to cluster" - run: ./scripts/ci/ci_deploy_app_to_kubernetes.sh + run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh - name: Cache virtualenv for kubernetes testing uses: actions/cache@v2 env: @@ -173,7 +173,7 @@ jobs: key: "${{ env.cache-name }}-${{ github.job }}-\ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') }}" - name: "Tests" - run: ./scripts/ci/ci_run_kubernetes_tests.sh + run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh - uses: actions/upload-artifact@v2 name: Upload KinD logs # Always run this, even if one of th previous steps failed. @@ -208,11 +208,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') with: python-version: '3.6' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" - run: ./scripts/ci/ci_run_airflow_testing.sh + run: ./scripts/ci/testing/ci_run_airflow_testing.sh tests-mysql: timeout-minutes: 80 @@ -240,11 +240,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') with: python-version: '3.x' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" - run: ./scripts/ci/ci_run_airflow_testing.sh + run: ./scripts/ci/testing/ci_run_airflow_testing.sh tests-sqlite: timeout-minutes: 80 @@ -270,11 +270,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') with: python-version: '3.x' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" - run: ./scripts/ci/ci_run_airflow_testing.sh + run: ./scripts/ci/testing/ci_run_airflow_testing.sh tests-quarantined: timeout-minutes: 80 @@ -303,11 +303,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') with: python-version: '3.x' - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Tests" - run: ./scripts/ci/ci_run_airflow_testing.sh + run: ./scripts/ci/testing/ci_run_airflow_testing.sh helm-tests: timeout-minutes: 5 @@ -337,11 +337,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') - uses: actions/checkout@master - uses: actions/setup-python@v1 - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Generate requirements" - run: ./scripts/ci/ci_generate_requirements.sh + run: ./scripts/ci/requirements/ci_generate_requirements.sh push-prod-images-to-github-cache: timeout-minutes: 80 @@ -364,11 +364,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') steps: - uses: actions/checkout@master - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build PROD images ${{ matrix.python-version }}" - run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh - name: "Push PROD images ${{ matrix.python-version }}" - run: ./scripts/ci/ci_push_production_images.sh + run: ./scripts/ci/images/ci_push_production_images.sh push-ci-images-to-github-cache: timeout-minutes: 40 @@ -395,8 +395,8 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') steps: - uses: actions/checkout@master - name: "Free space" - run: ./scripts/ci/ci_free_space_on_ci.sh + run: ./scripts/ci/tools/ci_free_space_on_ci.sh - name: "Build CI image" - run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - name: "Push CI image ${{ matrix.python-version }}" - run: ./scripts/ci/ci_push_ci_image.sh + run: ./scripts/ci/images/ci_push_ci_image.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99dbbe908c7ac5..eb91828ad24de7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -198,7 +198,7 @@ repos: - id: lint-dockerfile name: Lint dockerfile language: system - entry: "./scripts/ci/pre_commit_lint_dockerfile.sh" + entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh" files: ^Dockerfile.*$ pass_filenames: true - id: setup-order @@ -207,25 +207,25 @@ repos: files: ^setup.py$ pass_filenames: false require_serial: true - entry: tests/test_order_setup.py + entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py - id: update-breeze-file name: Update output of breeze command in BREEZE.rst - entry: "./scripts/ci/pre_commit_breeze_cmd_line.sh" + entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh" language: system files: ^BREEZE.rst$|^breeze$|^breeze-complete$ pass_filenames: false require_serial: true - id: update-local-yml-file name: Update mounts in the local yml file - entry: "./scripts/ci/pre_commit_local_yml_mounts.sh" + entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh" language: system files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml" pass_filenames: false require_serial: true - id: update-extras name: Update extras in documentation - entry: "./scripts/ci/pre_commit_update_extras.sh" - language: system + entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py + language: python files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$ pass_filenames: false require_serial: true @@ -265,14 +265,14 @@ repos: ^\sdef\s*\S*\(.*\):\s*\-\>\s*\S*.* # Matches -> return value syntax from Python3 )$ files: \.py$ - exclude: ^dev/ + exclude: ^dev|^scripts pass_filenames: true - id: python2-compile name: Compile code using python2 language: system entry: python2.7 -m py_compile files: \.py$ - exclude: ^dev/ + exclude: ^dev|^scripts pass_filenames: true require_serial: true - id: pydevd @@ -283,7 +283,7 @@ repos: pass_filenames: true - id: check-integrations name: Check if integration list is aligned - entry: ./scripts/ci/pre_commit_check_integrations.sh + entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh language: system pass_filenames: false files: ^common/_common_values.sh$|^breeze-complete$ @@ -295,13 +295,13 @@ repos: pass_filenames: true - id: build name: Check if image build is needed - entry: ./scripts/ci/pre_commit_ci_build.sh 3.5 false + entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false language: system always_run: true pass_filenames: false - id: check-apache-license name: Check if licenses are OK for Apache - entry: "./scripts/ci/pre_commit_check_license.sh" + entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh" language: system files: ^.*LICENSE.*$|^.*LICENCE.*$ pass_filenames: false @@ -309,28 +309,28 @@ repos: - id: airflow-config-yaml name: Checks for consistency between config.yml and default_config.cfg language: python + entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py files: "^airflow/config_templates/config.yml$|^airflow/config_templates/default_airflow.cfg$" pass_filenames: false - require_serial: false - entry: scripts/ci/pre_commit_yaml_to_cfg.py + require_serial: true additional_dependencies: ['pyyaml'] - id: mypy name: Run mypy language: system - entry: "./scripts/ci/pre_commit_mypy.sh" + entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh" files: \.py$ exclude: ^dev require_serial: true - id: flake8 name: Run flake8 language: system - entry: "./scripts/ci/pre_commit_flake8.sh" + entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh" files: \.py$ - exclude: ^dev/ + exclude: ^dev pass_filenames: true - id: bat-tests name: Run BATS bash tests for changed bash files language: system - entry: "./scripts/ci/pre_commit_bat_tests.sh" + entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh" files: ^breeze$|^breeze-complete$|\.sh$|\.bash$ pass_filenames: false diff --git a/.rat-excludes b/.rat-excludes index 6bef9643e12168..497d7ed509bc47 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -86,3 +86,4 @@ input_notebook.ipynb # .git might be a file in case of worktree .git +tmp diff --git a/BREEZE.rst b/BREEZE.rst index 338777bc4e76d4..435b21e7b59ef9 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -920,7 +920,7 @@ by the root user, you can fix the ownership of those files by running this scrip .. code-block:: - ./scripts/ci/ci_fix_ownership.sh + ./scripts/ci/tools/ci_fix_ownership.sh Mounting Local Sources to Breeze -------------------------------- diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 3cb2c5e7f0423e..b3b5978b80ec83 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -265,13 +265,12 @@ Running Static Code Checks via Scripts from the Host .................................................... You can trigger the static checks from the host environment, without entering the Docker container. To do -this, run the following scripts (the same is done in the CI builds): +this, run the following scripts: -* ``_ - checks the licenses. -* ``_ - checks that documentation can be built without warnings. -* ``_ - runs Flake8 source code style enforcement tool. -* ``_ - runs lint checker for the dockerfiles. -* ``_ - runs a check for mypy type annotation consistency. +* ``_ - checks the licenses. +* ``_ - runs Flake8 source code style enforcement tool. +* ``_ - runs lint checker for the dockerfiles. +* ``_ - runs a check for mypy type annotation consistency. The scripts may ask you to rebuild the images, if needed. @@ -314,8 +313,8 @@ On the host: .. code-block:: - ./scripts/ci/ci_mypy.sh ./airflow/example_dags/ + ./scripts/ci/static_checks/ci_mypy.sh ./airflow/example_dags/ .. code-block:: - ./scripts/ci/ci_mypy.sh ./airflow/example_dags/test_utils.py + ./scripts/ci/static_checks/ci_mypy.sh ./airflow/example_dags/test_utils.py diff --git a/TESTING.rst b/TESTING.rst index 7c761e85d99a29..02163fe4a3cea3 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -378,10 +378,10 @@ to run the tests manually one by one. Running kubernetes tests - ./scripts/ci/ci_run_kubernetes_tests.sh - runs all kubernetes tests - ./scripts/ci/ci_run_kubernetes_tests.sh TEST [TEST ...] - runs selected kubernetes tests (from kubernetes_tests folder) - ./scripts/ci/ci_run_kubernetes_tests.sh [-i|--interactive] - Activates virtual environment ready to run tests and drops you in - ./scripts/ci/ci_run_kubernetes_tests.sh [--help] - Prints this help message + ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh - runs all kubernetes tests + ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh TEST [TEST ...] - runs selected kubernetes tests (from kubernetes_tests folder) + ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [-i|--interactive] - Activates virtual environment ready to run tests and drops you in + ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [--help] - Prints this help message You can also run the same tests command with Breeze, using ``kind-cluster test`` command (to run all diff --git a/breeze b/breeze index a10ba9c5c9b680..abb95b75342b85 100755 --- a/breeze +++ b/breeze @@ -62,8 +62,8 @@ function setup_default_breeze_variables() { # load all the common functions here - those are the functions that are shared between Breeze # and CI scripts (CI scripts do not use Breeze as execution environment) - # shellcheck source=scripts/ci/_all_libs.sh - . "${SCRIPTS_CI_DIR}/_all_libs.sh" + # shellcheck source=scripts/ci/libraries/_all_libs.sh + . "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh" # We have different versions of images depending on the python version used. We keep up with the # Latest patch-level changes in Python (this is done automatically during CI builds) so we have diff --git a/docs/start_doc_server.sh b/docs/start_doc_server.sh index 26248ece7a9295..51cc7a77def2f4 100755 --- a/docs/start_doc_server.sh +++ b/docs/start_doc_server.sh @@ -16,8 +16,8 @@ # specific language governing permissions and limitations # under the License. -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -(cd "${MY_DIR}"/_build/html || exit; +DOCS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +(cd "${DOCS_DIR}"/_build/html || exit; # The below command works on both Python 2 and Python 3 python -m http.server 8000 && python -m SimpleHTTPServer 8000 ) diff --git a/hooks/build b/hooks/build index a98a9237bbee12..d3d3efb9f1c1f5 100755 --- a/hooks/build +++ b/hooks/build @@ -20,9 +20,9 @@ # on Travis CI to potentially rebuild (and refresh layers that # are not cached) Docker images that are used to run CI jobs -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +_HOOK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Dockerhub builds are run inside Docker container export SKIP_IN_CONTAINER_CHECK="true" -exec "${MY_DIR}/../scripts/ci/ci_build_dockerhub.sh" +exec "${_HOOK_DIR}/../scripts/ci/images/ci_build_dockerhub.sh" diff --git a/hooks/push b/hooks/push index 91cf0969ea6f34..84805b0d64e704 100755 --- a/hooks/push +++ b/hooks/push @@ -20,11 +20,6 @@ # and it is difficult to pass list of the built images from the build to push phase set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -echo "My dir: ${MY_DIR}" - - echo echo "Skip pushing the image. All images were built and pushed in the build hook already!" echo diff --git a/scripts/ci/ci_load_image_to_kind.sh b/scripts/ci/ci_load_image_to_kind.sh deleted file mode 100755 index dda1e387f72e26..00000000000000 --- a/scripts/ci/ci_load_image_to_kind.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" - -cd "${AIRFLOW_SOURCES}" || exit 1 - -export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"} -export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"} - -prepare_prod_build -echo -echo "Loading the ${AIRFLOW_PROD_IMAGE} to cluster ${KIND_CLUSTER_NAME} from docker" -echo -"${AIRFLOW_SOURCES}/.build/bin/kind" load docker-image --name "${KIND_CLUSTER_NAME}" "${AIRFLOW_PROD_IMAGE}" -echo -echo "Loaded the ${AIRFLOW_PROD_IMAGE} to cluster ${KIND_CLUSTER_NAME}" -echo diff --git a/scripts/ci/ci_perform_kind_cluster_operation.sh b/scripts/ci/ci_perform_kind_cluster_operation.sh deleted file mode 100755 index 4d3ddd27d9725d..00000000000000 --- a/scripts/ci/ci_perform_kind_cluster_operation.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" - -# adding trap to exiting trap -HANDLERS="$( trap -p EXIT | cut -f2 -d \' )" -# shellcheck disable=SC2064 -trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT - -get_environment_for_builds_on_ci -make_sure_kubernetes_tools_are_installed -initialize_kind_variables -perform_kind_cluster_operation "${@}" - -check_cluster_ready_for_airflow diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/docs/ci_docs.sh similarity index 92% rename from scripts/ci/ci_docs.sh rename to scripts/ci/docs/ci_docs.sh index 761e9a0cf344d3..847cb49f1a1b82 100755 --- a/scripts/ci/ci_docs.sh +++ b/scripts/ci/docs/ci_docs.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_docs() { docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \ diff --git a/scripts/ci/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh similarity index 95% rename from scripts/ci/ci_build_dockerhub.sh rename to scripts/ci/images/ci_build_dockerhub.sh index 97f9b8cdd732f7..99612a28e4d690 100755 --- a/scripts/ci/ci_build_dockerhub.sh +++ b/scripts/ci/images/ci_build_dockerhub.sh @@ -47,8 +47,8 @@ echo "DOCKER_TAG=${DOCKER_TAG}" echo "Detected PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}" echo -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" if [[ ${DOCKER_TAG} == *python*-ci ]]; then echo diff --git a/scripts/ci/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh similarity index 87% rename from scripts/ci/ci_prepare_ci_image_on_ci.sh rename to scripts/ci/images/ci_prepare_ci_image_on_ci.sh index 8ced22035004ea..5fd7913a8ad9f7 100755 --- a/scripts/ci/ci_prepare_ci_image_on_ci.sh +++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" build_ci_image_on_ci diff --git a/scripts/ci/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh similarity index 87% rename from scripts/ci/ci_prepare_prod_image_on_ci.sh rename to scripts/ci/images/ci_prepare_prod_image_on_ci.sh index 066a43b63318ce..ab4e7c0f640daa 100755 --- a/scripts/ci/ci_prepare_prod_image_on_ci.sh +++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" build_prod_image_on_ci diff --git a/scripts/ci/ci_push_ci_image.sh b/scripts/ci/images/ci_push_ci_image.sh similarity index 88% rename from scripts/ci/ci_push_ci_image.sh rename to scripts/ci/images/ci_push_ci_image.sh index 09e2a7aef50fdf..664d81a63872e8 100755 --- a/scripts/ci/ci_push_ci_image.sh +++ b/scripts/ci/images/ci_push_ci_image.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" prepare_ci_build diff --git a/scripts/ci/ci_push_production_images.sh b/scripts/ci/images/ci_push_production_images.sh similarity index 88% rename from scripts/ci/ci_push_production_images.sh rename to scripts/ci/images/ci_push_production_images.sh index d6c6e7d68f4396..db439d8a3c57e1 100755 --- a/scripts/ci/ci_push_production_images.sh +++ b/scripts/ci/images/ci_push_production_images.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" prepare_prod_build diff --git a/scripts/ci/in_container/_in_container_script_init.sh b/scripts/ci/in_container/_in_container_script_init.sh index 35400162f44328..50a558e80107ee 100755 --- a/scripts/ci/in_container/_in_container_script_init.sh +++ b/scripts/ci/in_container/_in_container_script_init.sh @@ -19,10 +19,10 @@ set -euo pipefail # This should only be sourced from in_container directory! -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +IN_CONTAINER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # shellcheck source=scripts/ci/in_container/_in_container_utils.sh -. "${MY_DIR}/_in_container_utils.sh" +. "${IN_CONTAINER_DIR}/_in_container_utils.sh" in_container_basic_sanity_check diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh index 0eb3a8a896a343..f2e94d4682ab81 100644 --- a/scripts/ci/in_container/_in_container_utils.sh +++ b/scripts/ci/in_container/_in_container_utils.sh @@ -155,7 +155,7 @@ function setup_kerberos() { PASS="airflow" KRB5_KTNAME=/etc/airflow.keytab - sudo cp "${MY_DIR}/krb5/krb5.conf" /etc/krb5.conf + sudo cp "${AIRFLOW_SOURCES}/scripts/ci/in_container/krb5/krb5.conf" /etc/krb5.conf echo -e "${PASS}\n${PASS}" | \ sudo kadmin -p "${ADMIN}/admin" -w "${PASS}" -q "addprinc -randkey airflow/${FQDN}" 2>&1 \ diff --git a/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh b/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh deleted file mode 100755 index a124fba263cc4e..00000000000000 --- a/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# Script to run flake8 on all code. Can be started from any working directory -# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" - -"${MY_DIR}/kubernetes/docker/rebuild_airflow_image.sh" -"${MY_DIR}/kubernetes/app/deploy_app.sh" diff --git a/scripts/ci/in_container/entrypoint_ci.sh b/scripts/ci/in_container/entrypoint_ci.sh index 4d1bf0c1b72753..eb1ff515bee925 100755 --- a/scripts/ci/in_container/entrypoint_ci.sh +++ b/scripts/ci/in_container/entrypoint_ci.sh @@ -22,7 +22,7 @@ fi # shellcheck source=scripts/ci/in_container/_in_container_script_init.sh . /opt/airflow/scripts/ci/in_container/_in_container_script_init.sh -AIRFLOW_SOURCES=$(cd "${MY_DIR}/../../.." || exit 1; pwd) +AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../../.." || exit 1; pwd) PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.6} BACKEND=${BACKEND:=sqlite} @@ -47,8 +47,8 @@ INSTALL_AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION:=""}" if [[ ${CI} == "false" ]]; then # Create links for useful CLI tools - # shellcheck source=scripts/ci/run_cli_tool.sh - source <(bash scripts/ci/run_cli_tool.sh) + # shellcheck source=scripts/ci/in_container/run_cli_tool.sh + source <(bash scripts/ci/in_container/run_cli_tool.sh) fi if [[ ${AIRFLOW_VERSION} == *1.10* || ${INSTALL_AIRFLOW_VERSION} == *1.10* ]]; then @@ -98,10 +98,10 @@ export PATH=${PATH}:${AIRFLOW_SOURCES} unset AIRFLOW__CORE__UNIT_TEST_MODE mkdir -pv "${AIRFLOW_HOME}/logs/" -cp -f "${MY_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg" +cp -f "${IN_CONTAINER_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg" set +e -"${MY_DIR}/check_environment.sh" +"${IN_CONTAINER_DIR}/check_environment.sh" ENVIRONMENT_EXIT_CODE=$? set -e if [[ ${ENVIRONMENT_EXIT_CODE} != 0 ]]; then @@ -150,7 +150,7 @@ done ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null # shellcheck source=scripts/ci/in_container/configure_environment.sh -. "${MY_DIR}/configure_environment.sh" +. "${IN_CONTAINER_DIR}/configure_environment.sh" cd "${AIRFLOW_SOURCES}" @@ -211,7 +211,7 @@ fi ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${TESTS_TO_RUN[@]}") if [[ ${RUN_SYSTEM_TESTS:="false"} == "true" ]]; then - "${MY_DIR}/run_system_tests.sh" "${ARGS[@]}" + "${IN_CONTAINER_DIR}/run_system_tests.sh" "${ARGS[@]}" else - "${MY_DIR}/run_ci_tests.sh" "${ARGS[@]}" + "${IN_CONTAINER_DIR}/run_ci_tests.sh" "${ARGS[@]}" fi diff --git a/scripts/ci/run_cli_tool.sh b/scripts/ci/in_container/run_cli_tool.sh similarity index 100% rename from scripts/ci/run_cli_tool.sh rename to scripts/ci/in_container/run_cli_tool.sh diff --git a/scripts/ci/in_container/run_system_tests.sh b/scripts/ci/in_container/run_system_tests.sh index 11dcf06436bf00..8cb3c3e440b361 100755 --- a/scripts/ci/in_container/run_system_tests.sh +++ b/scripts/ci/in_container/run_system_tests.sh @@ -20,10 +20,10 @@ # Bash sanity settings (error on exit, complain for undefined vars, error when pipe fails) set -euo pipefail -MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) +IN_CONTAINER_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) # shellcheck source=scripts/ci/in_container/_in_container_utils.sh -. "${MY_DIR}/_in_container_utils.sh" +. "${IN_CONTAINER_DIR}/_in_container_utils.sh" in_container_basic_sanity_check diff --git a/scripts/ci/ci_deploy_app_to_kubernetes.sh b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh similarity index 92% rename from scripts/ci/ci_deploy_app_to_kubernetes.sh rename to scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh index 307bf00def4a54..d4bb4b1e5b8561 100755 --- a/scripts/ci/ci_deploy_app_to_kubernetes.sh +++ b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" set -euo pipefail diff --git a/scripts/ci/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh similarity index 96% rename from scripts/ci/ci_run_kubernetes_tests.sh rename to scripts/ci/kubernetes/ci_run_kubernetes_tests.sh index 4d49e9e60c6d2a..3d8194a4bca507 100755 --- a/scripts/ci/ci_run_kubernetes_tests.sh +++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" # adding trap to exiting trap HANDLERS="$( trap -p EXIT | cut -f2 -d \' )" diff --git a/scripts/ci/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh similarity index 68% rename from scripts/ci/_all_libs.sh rename to scripts/ci/libraries/_all_libs.sh index 9869598b0de1fe..f70bbde8f6174b 100755 --- a/scripts/ci/_all_libs.sh +++ b/scripts/ci/libraries/_all_libs.sh @@ -16,34 +16,33 @@ # specific language governing permissions and limitations # under the License. -SCRIPTS_CI_DIR=$(dirname "${BASH_SOURCE[0]}") +LIBRARIES_DIR=$(dirname "${BASH_SOURCE[0]}") -# must be first to initialize arrays TODO: For sure? # shellcheck source=scripts/ci/libraries/_initialization.sh -. "${SCRIPTS_CI_DIR}"/libraries/_initialization.sh +. "${LIBRARIES_DIR}"/_initialization.sh # shellcheck source=scripts/ci/libraries/_sanity_checks.sh -. "${SCRIPTS_CI_DIR}"/libraries/_sanity_checks.sh +. "${LIBRARIES_DIR}"/_sanity_checks.sh # shellcheck source=scripts/ci/libraries/_build_images.sh -. "${SCRIPTS_CI_DIR}"/libraries/_build_images.sh +. "${LIBRARIES_DIR}"/_build_images.sh # shellcheck source=scripts/ci/libraries/_kind.sh -. "${SCRIPTS_CI_DIR}"/libraries/_kind.sh +. "${LIBRARIES_DIR}"/_kind.sh # shellcheck source=scripts/ci/libraries/_local_mounts.sh -. "${SCRIPTS_CI_DIR}"/libraries/_local_mounts.sh +. "${LIBRARIES_DIR}"/_local_mounts.sh # shellcheck source=scripts/ci/libraries/_md5sum.sh -. "${SCRIPTS_CI_DIR}"/libraries/_md5sum.sh +. "${LIBRARIES_DIR}"/_md5sum.sh # shellcheck source=scripts/ci/libraries/_parameters.sh -. "${SCRIPTS_CI_DIR}"/libraries/_parameters.sh +. "${LIBRARIES_DIR}"/_parameters.sh # shellcheck source=scripts/ci/libraries/_permissions.sh -. "${SCRIPTS_CI_DIR}"/libraries/_permissions.sh +. "${LIBRARIES_DIR}"/_permissions.sh # shellcheck source=scripts/ci/libraries/_push_pull_remove_images.sh -. "${SCRIPTS_CI_DIR}"/libraries/_push_pull_remove_images.sh +. "${LIBRARIES_DIR}"/_push_pull_remove_images.sh # shellcheck source=scripts/ci/libraries/_runs.sh -. "${SCRIPTS_CI_DIR}"/libraries/_runs.sh +. "${LIBRARIES_DIR}"/_runs.sh # shellcheck source=scripts/ci/libraries/_spinner.sh -. "${SCRIPTS_CI_DIR}"/libraries/_spinner.sh +. "${LIBRARIES_DIR}"/_spinner.sh # shellcheck source=scripts/ci/libraries/_start_end.sh -. "${SCRIPTS_CI_DIR}"/libraries/_start_end.sh +. "${LIBRARIES_DIR}"/_start_end.sh # shellcheck source=scripts/ci/libraries/_verbosity.sh -. "${SCRIPTS_CI_DIR}"/libraries/_verbosity.sh +. "${LIBRARIES_DIR}"/_verbosity.sh diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 352975b6b07cb6..01eac17046b32c 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -421,7 +421,7 @@ function rebuild_ci_image_if_needed() { if [[ ${SYSTEM} != "Darwin" ]]; then ROOT_FILES_COUNT=$(find "airflow" "tests" -user root | wc -l | xargs) if [[ ${ROOT_FILES_COUNT} != "0" ]]; then - ./scripts/ci/ci_fix_ownership.sh + ./scripts/ci/tools/ci_fix_ownership.sh fi fi print_info diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 5f2a74251d350a..d0b14a7127d704 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -33,10 +33,6 @@ function initialize_common_environment { # shellcheck disable=SC2034 FILES_TO_CLEANUP_ON_EXIT=() - # Sets to where airflow sources are located - AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${MY_DIR}/../../" && pwd)} - export AIRFLOW_SOURCES - # Sets to the build cache directory - status of build and convenience scripts are stored there BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build" export BUILD_CACHE_DIR @@ -172,7 +168,7 @@ function initialize_common_environment { fi # Read airflow version from the version.py - AIRFLOW_VERSION=$(grep version "airflow/version.py" | awk '{print $3}' | sed "s/['+]//g") + AIRFLOW_VERSION=$(grep version "${AIRFLOW_SOURCES}/airflow/version.py" | awk '{print $3}' | sed "s/['+]//g") export AIRFLOW_VERSION # default version of python used to tag the "master" and "latest" images in DockerHub diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index f24b3aa2a564c1..2cf43c35fb670f 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -125,6 +125,16 @@ function delete_cluster() { } function perform_kind_cluster_operation() { + ALLOWED_KIND_OPERATIONS="[ start restart stop deploy test shell ]" + + set +u + if [[ -z "${1}" ]]; then + echo >&2 + echo >&2 "Operation must be provided as first parameter. One of: ${ALLOWED_KIND_OPERATIONS}" + echo >&2 + exit 1 + fi + set -u OPERATION="${1}" ALL_CLUSTERS=$(kind get clusters || true) @@ -181,17 +191,16 @@ function perform_kind_cluster_operation() { echo echo "Testing with KinD" echo - "${AIRFLOW_SOURCES}/scripts/ci/ci_run_kubernetes_tests.sh" + "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh" elif [[ ${OPERATION} == "shell" ]]; then echo echo "Entering an interactive shell for kubernetes testing" echo - "${AIRFLOW_SOURCES}/scripts/ci/ci_run_kubernetes_tests.sh" "-i" + "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh" "-i" else - echo - echo "Wrong cluster operation: ${OPERATION}. Should be one of:" - echo "${FORMATTED_KIND_OPERATIONS}" - echo + echo >&2 + echo >&2 "Wrong cluster operation: ${OPERATION}. Should be one of: ${ALLOWED_KIND_OPERATIONS}" + echo >&2 exit 1 fi else @@ -208,15 +217,14 @@ function perform_kind_cluster_operation() { create_cluster elif [[ ${OPERATION} == "stop" || ${OEPRATON} == "deploy" || \ ${OPERATION} == "test" || ${OPERATION} == "shell" ]]; then - echo - echo "Cluster ${KIND_CLUSTER_NAME} does not exist. It should exist for ${OPERATION} operation" - echo + echo >&2 + echo >&2 "Cluster ${KIND_CLUSTER_NAME} does not exist. It should exist for ${OPERATION} operation" + echo >&2 exit 1 else - echo - echo "Wrong cluster operation: ${OPERATION}. Should be one of:" - echo "${FORMATTED_KIND_OPERATIONS}" - echo + echo >&2 + echo >&2 "Wrong cluster operation: ${OPERATION}. Should be one of ${ALLOWED_KIND_OPERATIONS}" + echo >&2 exit 1 fi fi @@ -262,9 +270,9 @@ function forward_port_to_kind_webserver() { set +e while ! curl http://localhost:8080/health -s | grep -q healthy; do if [[ ${num_tries} == 6 ]]; then - echo - echo "ERROR! Could not setup a forward port to Airflow's webserver after ${num_tries}! Exiting." - echo + echo >&2 + echo >&2 "ERROR! Could not setup a forward port to Airflow's webserver after ${num_tries}! Exiting." + echo >&2 exit 1 fi echo diff --git a/scripts/ci/_script_init.sh b/scripts/ci/libraries/_script_init.sh similarity index 73% rename from scripts/ci/_script_init.sh rename to scripts/ci/libraries/_script_init.sh index 2acbf105c62dce..804037eac0083e 100755 --- a/scripts/ci/_script_init.sh +++ b/scripts/ci/libraries/_script_init.sh @@ -18,17 +18,18 @@ set -euo pipefail -# This should only be sourced from CI directory! +_CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") -SCRIPTS_CI_DIR="$( dirname "${BASH_SOURCE[0]}" )" +SCRIPTS_CI_DIR="$(cd "${_CURRENT_DIR}"/.. && pwd)" export SCRIPTS_CI_DIR -# shellcheck source=scripts/ci/_all_libs.sh -. "${SCRIPTS_CI_DIR}"/_all_libs.sh +# Sets to where airflow sources are located +AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${SCRIPTS_CI_DIR}/../../" && pwd)} +export AIRFLOW_SOURCES -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export MY_DIR +# shellcheck source=scripts/ci/libraries/_all_libs.sh +. "${SCRIPTS_CI_DIR}"/libraries/_all_libs.sh initialize_common_environment diff --git a/scripts/ci/minikdc.properties b/scripts/ci/minikdc.properties deleted file mode 100644 index c70ff8448bf6da..00000000000000 --- a/scripts/ci/minikdc.properties +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -org.name=TEST -org.domain=LOCAL -kdc.bind.address=localhost -kdc.port=8888 -instance=DefaultKrbServer -max.ticket.lifetime=86400000 -max.renewable.lifetime=604800000 -transport=TCP -debug=true diff --git a/scripts/ci/pre_commit_bat_tests.sh b/scripts/ci/pre_commit/pre_commit_bat_tests.sh similarity index 87% rename from scripts/ci/pre_commit_bat_tests.sh rename to scripts/ci/pre_commit/pre_commit_bat_tests.sh index 4ae72c336bf36d..b820c7e60fd9d6 100755 --- a/scripts/ci/pre_commit_bat_tests.sh +++ b/scripts/ci/pre_commit/pre_commit_bat_tests.sh @@ -24,5 +24,5 @@ else PARAMS=("${@}") fi -# shellcheck source=scripts/ci/ci_bat_tests.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_bat_tests.sh" "${PARAMS[@]}" +# shellcheck source=scripts/ci/static_checks/ci_bat_tests.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_bat_tests.sh" "${PARAMS[@]}" diff --git a/scripts/ci/pre_commit_breeze_cmd_line.sh b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh similarity index 89% rename from scripts/ci/pre_commit_breeze_cmd_line.sh rename to scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh index eeac2e6ecc7e24..4d62cc1ed068e5 100755 --- a/scripts/ci/pre_commit_breeze_cmd_line.sh +++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh @@ -18,12 +18,14 @@ set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd); +cd "${AIRFLOW_SOURCES}" || exit 1 + + TMP_FILE=$(mktemp) TMP_OUTPUT=$(mktemp) -cd "${MY_DIR}/../../" || exit; - echo " .. code-block:: text " >"${TMP_FILE}" @@ -46,7 +48,7 @@ if (( MAX_LEN > MAX_SCREEN_WIDTH + 2 )); then exit 1 fi -BREEZE_RST_FILE="${MY_DIR}/../../BREEZE.rst" +BREEZE_RST_FILE="${AIRFLOW_SOURCES}/BREEZE.rst" LEAD='^ \.\. START BREEZE HELP MARKER$' TAIL='^ \.\. END BREEZE HELP MARKER$' diff --git a/scripts/ci/pre_commit_check_integrations.sh b/scripts/ci/pre_commit/pre_commit_check_integrations.sh similarity index 81% rename from scripts/ci/pre_commit_check_integrations.sh rename to scripts/ci/pre_commit/pre_commit_check_integrations.sh index 69ace38a2a8006..687194174fc9e0 100755 --- a/scripts/ci/pre_commit_check_integrations.sh +++ b/scripts/ci/pre_commit/pre_commit_check_integrations.sh @@ -17,12 +17,12 @@ # under the License. set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd); +cd "${AIRFLOW_SOURCES}" || exit 1 -cd "${MY_DIR}/../../" || exit; - -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" . breeze-complete diff --git a/scripts/ci/pre_commit_check_license.sh b/scripts/ci/pre_commit/pre_commit_check_license.sh similarity index 92% rename from scripts/ci/pre_commit_check_license.sh rename to scripts/ci/pre_commit/pre_commit_check_license.sh index 4ab6964c57bbd1..109df313ef4a54 100755 --- a/scripts/ci/pre_commit_check_license.sh +++ b/scripts/ci/pre_commit/pre_commit_check_license.sh @@ -21,5 +21,5 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} export REMEMBER_LAST_ANSWER="true" # Hide lines between ****/**** (detailed list of files) -"$( dirname "${BASH_SOURCE[0]}" )/ci_check_license.sh" 2>&1 | \ +"$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_check_license.sh" 2>&1 | \ (sed "/Files with Apache License headers will be marked AL.*$/,/^\**$/d" || true) diff --git a/scripts/ci/pre_commit/pre_commit_check_order_setup.py b/scripts/ci/pre_commit/pre_commit_check_order_setup.py new file mode 100755 index 00000000000000..c833e22e6d24ea --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_check_order_setup.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Test for an order of dependencies in setup.py +""" + +import os +import re +import sys +from os.path import abspath, dirname +from typing import List + +errors = [] + + +def _check_list_sorted(the_list: List[str], message: str) -> None: + sorted_list = sorted(the_list) + if the_list == sorted_list: + print(f"{message} is ok") + return + i = 0 + while sorted_list[i] == the_list[i]: + i += 1 + print(f"{message} NOK") + errors.append(f"ERROR in {message}. First wrongly sorted element" + f" {the_list[i]}. Should be {sorted_list[i]}") + + +def setup() -> str: + setup_py_file_path = abspath(os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir, 'setup.py')) + with open(setup_py_file_path) as setup_file: + setup_context = setup_file.read() + return setup_context + + +def check_main_dependent_group(setup_context: str) -> None: + """ + Test for an order of dependencies groups between mark + '# Start dependencies group' and '# End dependencies group' in setup.py + """ + pattern_main_dependent_group = re.compile( + '# Start dependencies group\n(.*)# End dependencies group', re.DOTALL) + main_dependent_group = pattern_main_dependent_group.findall(setup_context)[0] + + pattern_sub_dependent = re.compile(' = \\[.*?\\]\n', re.DOTALL) + main_dependent = pattern_sub_dependent.sub(',', main_dependent_group) + + src = main_dependent.strip(',').split(',') + _check_list_sorted(src, "Order of dependencies") + + +def check_sub_dependent_group(setup_context: str) -> None: + r""" + Test for an order of each dependencies groups declare like + `^dependent_group_name = [.*?]\n` in setup.py + """ + pattern_dependent_group_name = re.compile('^(\\w+) = \\[', re.MULTILINE) + dependent_group_names = pattern_dependent_group_name.findall(setup_context) + + pattern_dependent_version = re.compile('[~|><=;].*') + + for group_name in dependent_group_names: + pattern_sub_dependent = re.compile( + '{group_name} = \\[(.*?)\\]'.format(group_name=group_name), re.DOTALL) + sub_dependent = pattern_sub_dependent.findall(setup_context)[0] + pattern_dependent = re.compile('\'(.*?)\'') + dependent = pattern_dependent.findall(sub_dependent) + + src = [pattern_dependent_version.sub('', p) for p in dependent] + _check_list_sorted(src, f"Order of sub-dependencies group: {group_name}") + + +def check_alias_dependent_group(setup_context: str) -> None: + """ + Test for an order of each dependencies groups declare like + `alias_dependent_group = dependent_group_1 + ... + dependent_group_n` in setup.py + """ + pattern = re.compile('^\\w+ = (\\w+ \\+.*)', re.MULTILINE) + dependents = pattern.findall(setup_context) + + for dependent in dependents: + src = dependent.split(' + ') + _check_list_sorted(src, f"Order of alias dependencies group: {dependent}") + + +def check_install_and_setup_requires(setup_context: str) -> None: + """ + Test for an order of dependencies in function do_setup section + install_requires and setup_requires in setup.py + """ + pattern_install_and_setup_requires = re.compile( + '(setup_requires) ?= ?\\[(.*?)\\]', re.DOTALL) + install_and_setup_requires = pattern_install_and_setup_requires.findall(setup_context) + + for dependent_requires in install_and_setup_requires: + pattern_dependent = re.compile('\'(.*?)\'') + dependent = pattern_dependent.findall(dependent_requires[1]) + pattern_dependent_version = re.compile('[~|><=;].*') + + src = [pattern_dependent_version.sub('', p) for p in dependent] + _check_list_sorted(src, f"Order of dependencies in do_setup section: {dependent_requires[0]}") + + +if __name__ == '__main__': + setup_context_main = setup() + check_main_dependent_group(setup_context_main) + check_alias_dependent_group(setup_context_main) + check_sub_dependent_group(setup_context_main) + check_install_and_setup_requires(setup_context_main) + + print() + print() + for error in errors: + print(error) + + print() + + if errors: + sys.exit(1) diff --git a/scripts/ci/pre_commit_ci_build.sh b/scripts/ci/pre_commit/pre_commit_ci_build.sh similarity index 88% rename from scripts/ci/pre_commit_ci_build.sh rename to scripts/ci/pre_commit/pre_commit_ci_build.sh index 6f25c39a4fc567..861785bcc5157a 100755 --- a/scripts/ci/pre_commit_ci_build.sh +++ b/scripts/ci/pre_commit/pre_commit_ci_build.sh @@ -18,8 +18,8 @@ export PYTHON_MAJOR_MINOR_VERSION="${1}" export REMEMBER_LAST_ANSWER="${2}" -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" forget_last_answer diff --git a/scripts/ci/pre_commit_flake8.sh b/scripts/ci/pre_commit/pre_commit_flake8.sh similarity index 87% rename from scripts/ci/pre_commit_flake8.sh rename to scripts/ci/pre_commit/pre_commit_flake8.sh index dd2a3da0b52836..95f1b987bceeb9 100755 --- a/scripts/ci/pre_commit_flake8.sh +++ b/scripts/ci/pre_commit/pre_commit_flake8.sh @@ -18,5 +18,5 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} export REMEMBER_LAST_ANSWER="true" -# shellcheck source=scripts/ci/ci_flake8.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_flake8.sh" "${@}" +# shellcheck source=scripts/ci/static_checks/ci_flake8.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_flake8.sh" "${@}" diff --git a/scripts/ci/pre_commit_generate_requirements.sh b/scripts/ci/pre_commit/pre_commit_generate_requirements.sh similarity index 85% rename from scripts/ci/pre_commit_generate_requirements.sh rename to scripts/ci/pre_commit/pre_commit_generate_requirements.sh index f9e952e8471b1d..d0c2deb34cf025 100755 --- a/scripts/ci/pre_commit_generate_requirements.sh +++ b/scripts/ci/pre_commit/pre_commit_generate_requirements.sh @@ -20,5 +20,5 @@ export REMEMBER_LAST_ANSWER="true" export PYTHON_MAJOR_MINOR_VERSION="${1}" -# shellcheck source=scripts/ci/ci_generate_requirements.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_generate_requirements.sh" +# shellcheck source=scripts/ci/requirements/ci_generate_requirements.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../generate_requirements/ci_generate_requirements.sh" diff --git a/tests/insert_extras.py b/scripts/ci/pre_commit/pre_commit_insert_extras.py old mode 100644 new mode 100755 similarity index 93% rename from tests/insert_extras.py rename to scripts/ci/pre_commit/pre_commit_insert_extras.py index b38b543214156a..9c441df8db5d60 --- a/tests/insert_extras.py +++ b/scripts/ci/pre_commit/pre_commit_insert_extras.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -20,8 +21,9 @@ from textwrap import wrap from typing import List -AIRFLOW_SOURCES_DIR = os.path.join(dirname(__file__), os.pardir) +AIRFLOW_SOURCES_DIR = os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir) +sys.path.insert(0, AIRFLOW_SOURCES_DIR) # flake8: noqa: F401 # pylint: disable=wrong-import-position from setup import EXTRAS_REQUIREMENTS # isort:skip diff --git a/scripts/ci/pre_commit_lint_dockerfile.sh b/scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh similarity index 84% rename from scripts/ci/pre_commit_lint_dockerfile.sh rename to scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh index bce941b120b39d..857c27e9f35440 100755 --- a/scripts/ci/pre_commit_lint_dockerfile.sh +++ b/scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh @@ -17,5 +17,5 @@ # under the License. export REMEMBER_LAST_ANSWER="true" -# shellcheck source=scripts/ci/ci_lint_dockerfile.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_lint_dockerfile.sh" "${@}" +# shellcheck source=scripts/ci/static_checks/ci_lint_dockerfile.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_lint_dockerfile.sh" "${@}" diff --git a/scripts/ci/pre_commit_local_yml_mounts.sh b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh similarity index 86% rename from scripts/ci/pre_commit_local_yml_mounts.sh rename to scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh index b638a08b65362a..43ece7e949a7fa 100755 --- a/scripts/ci/pre_commit_local_yml_mounts.sh +++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh @@ -18,17 +18,15 @@ set -euo pipefail -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" TMP_OUTPUT=$(mktemp) # Remove temp file if it's hanging around trap 'rm -rf -- "${TMP_OUTPUT}" 2>/dev/null' EXIT -LOCAL_YML_FILE="${MY_DIR}/docker-compose/local.yml" +LOCAL_YML_FILE="${AIRFLOW_SOURCES}/scripts/ci/docker-compose/local.yml" LEAD=' # START automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh' TAIL=' # END automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh' diff --git a/scripts/ci/pre_commit_mypy.sh b/scripts/ci/pre_commit/pre_commit_mypy.sh similarity index 87% rename from scripts/ci/pre_commit_mypy.sh rename to scripts/ci/pre_commit/pre_commit_mypy.sh index 6f0025f3485c6d..f5ab2541a9b9f8 100755 --- a/scripts/ci/pre_commit_mypy.sh +++ b/scripts/ci/pre_commit/pre_commit_mypy.sh @@ -18,5 +18,5 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} export REMEMBER_LAST_ANSWER="true" -# shellcheck source=scripts/ci/ci_mypy.sh -. "$( dirname "${BASH_SOURCE[0]}" )/ci_mypy.sh" "${@}" +# shellcheck source=scripts/ci/static_checks/ci_mypy.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../static_checks/ci_mypy.sh" "${@}" diff --git a/scripts/ci/pre_commit_yaml_to_cfg.py b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py similarity index 97% rename from scripts/ci/pre_commit_yaml_to_cfg.py rename to scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py index b8268647e32499..a578689f6180f6 100755 --- a/scripts/ci/pre_commit_yaml_to_cfg.py +++ b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -127,7 +126,9 @@ def write_config(yaml_config_file_path, default_cfg_file_path): if __name__ == '__main__': airflow_config_dir = os.path.join( - os.path.dirname(__file__), "../../airflow/config_templates") + os.path.dirname(__file__), + os.pardir, os.pardir, os.pardir, + "airflow", "config_templates") airflow_default_config_path = os.path.join(airflow_config_dir, "default_airflow.cfg") airflow_config_yaml_file_path = os.path.join(airflow_config_dir, "config.yml") diff --git a/scripts/ci/pre_commit_update_extras.sh b/scripts/ci/pre_commit_update_extras.sh deleted file mode 100755 index be43a91f4ab078..00000000000000 --- a/scripts/ci/pre_commit_update_extras.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -cd "${MY_DIR}/../../" || exit; - -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" - -PYTHONPATH="$(pwd)" -export PYTHONPATH - -python3 tests/insert_extras.py diff --git a/scripts/ci/ci_generate_requirements.sh b/scripts/ci/requirements/ci_generate_requirements.sh similarity index 88% rename from scripts/ci/ci_generate_requirements.sh rename to scripts/ci/requirements/ci_generate_requirements.sh index f55799ff84b2b1..5cc4a0e09ab478 100755 --- a/scripts/ci/ci_generate_requirements.sh +++ b/scripts/ci/requirements/ci_generate_requirements.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" get_environment_for_builds_on_ci diff --git a/scripts/ci/ci_bat_tests.sh b/scripts/ci/static_checks/ci_bat_tests.sh similarity index 90% rename from scripts/ci/ci_bat_tests.sh rename to scripts/ci/static_checks/ci_bat_tests.sh index 25011ce2c6c4c8..e50c55941ed009 100755 --- a/scripts/ci/ci_bat_tests.sh +++ b/scripts/ci/static_checks/ci_bat_tests.sh @@ -16,6 +16,9 @@ # specific language governing permissions and limitations # under the License. +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + function run_bats_tests() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then diff --git a/scripts/ci/ci_check_license.sh b/scripts/ci/static_checks/ci_check_license.sh similarity index 94% rename from scripts/ci/ci_check_license.sh rename to scripts/ci/static_checks/ci_check_license.sh index da5aebd94f0a5b..3d887c4219f480 100755 --- a/scripts/ci/ci_check_license.sh +++ b/scripts/ci/static_checks/ci_check_license.sh @@ -18,8 +18,8 @@ export MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="true" export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_check_license() { echo diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/static_checks/ci_flake8.sh similarity index 94% rename from scripts/ci/ci_flake8.sh rename to scripts/ci/static_checks/ci_flake8.sh index 33504c066c56f7..4ebd060328899a 100755 --- a/scripts/ci/ci_flake8.sh +++ b/scripts/ci/static_checks/ci_flake8.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_flake8() { FILES=("$@") diff --git a/scripts/ci/ci_lint_dockerfile.sh b/scripts/ci/static_checks/ci_lint_dockerfile.sh similarity index 92% rename from scripts/ci/ci_lint_dockerfile.sh rename to scripts/ci/static_checks/ci_lint_dockerfile.sh index 29a5f685e1c617..2e48043f5c78e6 100755 --- a/scripts/ci/ci_lint_dockerfile.sh +++ b/scripts/ci/static_checks/ci_lint_dockerfile.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_docker_lint() { FILES=("$@") diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/static_checks/ci_mypy.sh similarity index 93% rename from scripts/ci/ci_mypy.sh rename to scripts/ci/static_checks/ci_mypy.sh index 8cc3028970ac5c..b6fc56e5e1c555 100755 --- a/scripts/ci/ci_mypy.sh +++ b/scripts/ci/static_checks/ci_mypy.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=3.6 -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" function run_mypy() { FILES=("$@") diff --git a/scripts/ci/ci_run_static_checks.sh b/scripts/ci/static_checks/ci_run_static_checks.sh similarity index 91% rename from scripts/ci/ci_run_static_checks.sh rename to scripts/ci/static_checks/ci_run_static_checks.sh index bfdae1ab8e6705..6b7f12455b958e 100755 --- a/scripts/ci/ci_run_static_checks.sh +++ b/scripts/ci/static_checks/ci_run_static_checks.sh @@ -17,8 +17,8 @@ # under the License. export PYTHON_MAJOR_MINOR_VERSION=3.6 -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then echo diff --git a/scripts/ci/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh similarity index 82% rename from scripts/ci/ci_run_airflow_testing.sh rename to scripts/ci/testing/ci_run_airflow_testing.sh index 09dfb7d3a7b79c..79499961aa4b66 100755 --- a/scripts/ci/ci_run_airflow_testing.sh +++ b/scripts/ci/testing/ci_run_airflow_testing.sh @@ -17,28 +17,29 @@ # under the License. export VERBOSE=${VERBOSE:="false"} +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then + echo + echo "Skipping running tests !!!!!" + echo + exit +fi + + function run_airflow_testing_in_docker() { set +u # shellcheck disable=SC2016 docker-compose --log-level INFO \ - -f "${MY_DIR}/docker-compose/base.yml" \ - -f "${MY_DIR}/docker-compose/backend-${BACKEND}.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml" \ "${INTEGRATIONS[@]}" \ "${DOCKER_COMPOSE_LOCAL[@]}" \ run airflow "${@}" set -u } -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" - -if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then - echo - echo "Skipping running tests !!!!!" - echo - exit -fi - get_environment_for_builds_on_ci prepare_ci_build @@ -66,17 +67,17 @@ export FORWARD_CREDENTIALS=${FORWARD_CREDENTIALS:="false"} export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} if [[ ${MOUNT_LOCAL_SOURCES} == "true" ]]; then - DOCKER_COMPOSE_LOCAL=("-f" "${MY_DIR}/docker-compose/local.yml") + DOCKER_COMPOSE_LOCAL=("-f" "${SCRIPTS_CI_DIR}/docker-compose/local.yml") else DOCKER_COMPOSE_LOCAL=() fi if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${MY_DIR}/docker-compose/forward-credentials.yml") + DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml") fi if [[ ${INSTALL_AIRFLOW_VERSION} != "" || ${INSTALL_AIRFLOW_REFERENCE} != "" ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${MY_DIR}/docker-compose/remove-sources.yml") + DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml") fi echo @@ -99,7 +100,7 @@ fi for _INT in ${ENABLED_INTEGRATIONS} do INTEGRATIONS+=("-f") - INTEGRATIONS+=("${MY_DIR}/docker-compose/integration-${_INT}.yml") + INTEGRATIONS+=("${SCRIPTS_CI_DIR}/docker-compose/integration-${_INT}.yml") done RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""} diff --git a/scripts/ci/ci_count_changed_files.sh b/scripts/ci/tools/ci_count_changed_files.sh similarity index 92% rename from scripts/ci/ci_count_changed_files.sh rename to scripts/ci/tools/ci_count_changed_files.sh index d1ccd4bc3cb9a6..e3d2c9eb0c76a1 100755 --- a/scripts/ci/ci_count_changed_files.sh +++ b/scripts/ci/tools/ci_count_changed_files.sh @@ -22,8 +22,8 @@ # $1: Revision to compare # $2: Pattern to match -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" get_environment_for_builds_on_ci diff --git a/scripts/ci/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh similarity index 82% rename from scripts/ci/ci_fix_ownership.sh rename to scripts/ci/tools/ci_fix_ownership.sh index 7e85152a7c43fa..8cde42dc55c52e 100755 --- a/scripts/ci/ci_fix_ownership.sh +++ b/scripts/ci/tools/ci_fix_ownership.sh @@ -21,8 +21,8 @@ # export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6} -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" export AIRFLOW_CI_IMAGE=\ ${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci @@ -39,7 +39,7 @@ export HOST_OS export BACKEND="sqlite" docker-compose \ - -f "${MY_DIR}/docker-compose/base.yml" \ - -f "${MY_DIR}/docker-compose/local.yml" \ - -f "${MY_DIR}/docker-compose/forward-credentials.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/local.yml" \ + -f "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml" \ run airflow /opt/airflow/scripts/ci/in_container/run_fix_ownership.sh diff --git a/scripts/ci/ci_free_space_on_ci.sh b/scripts/ci/tools/ci_free_space_on_ci.sh similarity index 87% rename from scripts/ci/ci_free_space_on_ci.sh rename to scripts/ci/tools/ci_free_space_on_ci.sh index 5d8f8510a3b080..a50add7dbff85a 100755 --- a/scripts/ci/ci_free_space_on_ci.sh +++ b/scripts/ci/tools/ci_free_space_on_ci.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" sudo swapoff -a sudo rm -f /swapfile diff --git a/tests/bats/bats_utils.bash b/tests/bats/bats_utils.bash index 7a29da94322710..c06b1827634220 100644 --- a/tests/bats/bats_utils.bash +++ b/tests/bats/bats_utils.bash @@ -18,5 +18,5 @@ AIRFLOW_SOURCES=$(pwd) export AIRFLOW_SOURCES export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci -# shellcheck source=scripts/ci/_all_libs.sh -source "${SCRIPTS_CI_DIR}/_all_libs.sh" +# shellcheck source=scripts/ci/libraries/_all_libs.sh +source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh" diff --git a/tests/test_order_setup.py b/tests/test_order_setup.py deleted file mode 100755 index 5c40ea3b41e767..00000000000000 --- a/tests/test_order_setup.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Test for an order of dependencies in setup.py -""" - -import os -import re -import unittest - - -class TestOrderSetup(unittest.TestCase): - - def setUp(self): - current_dir = os.path.dirname(os.path.abspath(__file__)) - parent_dir = os.path.dirname(current_dir) - self.setup_file = open('{parent_dir}/setup.py'.format(parent_dir=parent_dir)) - self.setup_context = self.setup_file.read() - - def tearDown(self): - self.setup_file.close() - - def test_main_dependent_group(self): - """ - Test for an order of dependencies groups between mark - '# Start dependencies group' and '# End dependencies group' in setup.py - """ - pattern_main_dependent_group = re.compile( - '# Start dependencies group\n(.*)# End dependencies group', re.DOTALL) - main_dependent_group = pattern_main_dependent_group.findall(self.setup_context)[0] - - pattern_sub_dependent = re.compile(' = \\[.*?\\]\n', re.DOTALL) - main_dependent = pattern_sub_dependent.sub(',', main_dependent_group) - - src = main_dependent.strip(',').split(',') - alphabetical = sorted(src) - self.assertListEqual(alphabetical, src) - - def test_sub_dependent_group(self): - """ - Test for an order of each dependencies groups declare like - `^dependent_group_name = [.*?]\n` in setup.py - """ - pattern_dependent_group_name = re.compile('^(\\w+) = \\[', re.MULTILINE) - dependent_group_names = pattern_dependent_group_name.findall(self.setup_context) - - pattern_dependent_version = re.compile('[~|>|<|=|;].*') - for group_name in dependent_group_names: - pattern_sub_dependent = re.compile( - '{group_name} = \\[(.*?)\\]'.format(group_name=group_name), re.DOTALL) - sub_dependent = pattern_sub_dependent.findall(self.setup_context)[0] - pattern_dependent = re.compile('\'(.*?)\'') - dependent = pattern_dependent.findall(sub_dependent) - - src = [pattern_dependent_version.sub('', p) for p in dependent] - alphabetical = sorted(src) - self.assertListEqual(alphabetical, src) - - def test_alias_dependent_group(self): - """ - Test for an order of each dependencies groups declare like - `alias_dependent_group = dependent_group_1 + ... + dependent_group_n` in setup.py - """ - pattern = re.compile('^\\w+ = (\\w+ \\+.*)', re.MULTILINE) - dependents = pattern.findall(self.setup_context) - for dependent in dependents: - src = dependent.split(' + ') - alphabetical = sorted(src) - self.assertListEqual(alphabetical, src) - - def test_devel_all(self): - """ - Test for an order of dependencies groups - devel_all = (dependent_group_1 + ... + dependent_group_n) in setup.py - """ - pattern = re.compile('devel_all = \\((.*?)\\)', re.DOTALL) - dependent = pattern.findall(self.setup_context)[0] - pattern_new_line = re.compile('\\n *') - - src = pattern_new_line.sub(' ', dependent).split(' + ') - alphabetical = sorted(src) - self.assertListEqual(alphabetical, src) - - def test_install_and_setup_requires(self): - """ - Test for an order of dependencies in function do_setup section - install_requires and setup_requires in setup.py - """ - pattern_install_and_setup_requires = re.compile( - '(INSTALL_REQUIREMENTS|setup_requires) ?= ?\\[(.*?)\\]', re.DOTALL) - install_and_setup_requires = pattern_install_and_setup_requires.findall(self.setup_context) - - for dependent_requires in install_and_setup_requires: - pattern_dependent = re.compile('\'(.*?)\'') - dependent = pattern_dependent.findall(dependent_requires[1]) - pattern_dependent_version = re.compile('[~|>|<|=|;].*') - - src = [pattern_dependent_version.sub('', p) for p in dependent] - alphabetical = sorted(src) - self.assertListEqual(alphabetical, src) - - def test_extras_require(self): - """ - Test for an order of dependencies in function do_setup section - extras_require in setup.py - """ - pattern_extras_requires = re.compile('EXTRAS_REQUIREMENTS = \\{(.*?)\\}', re.DOTALL) - extras_requires = pattern_extras_requires.findall(self.setup_context)[0] - - pattern_dependent = re.compile('\'(.*?)\'') - src = pattern_dependent.findall(extras_requires) - alphabetical = sorted(src) - self.assertListEqual(alphabetical, src) - - -if __name__ == '__main__': - unittest.main(verbosity=2)