Skip to content

Commit

Permalink
Group CI scripts in subdirectories (apache#9653)
Browse files Browse the repository at this point in the history 
Reviewed the scripts and removed some of the old unused ones.
  • Loading branch information
@potiuk
potiuk authored Jul 16, 2020
1 parent 473f506 commit faec41e
Show file tree
Hide file tree
Showing 73 changed files with 289 additions and 429 deletions.
76 changes: 38 additions & 38 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,17 +62,17 @@ jobs:
path: ~/.cache/pre-commit
key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Static checks"
if: success()
env:
PYTHON_VERSION: 3.6
run: |
python -m pip install pre-commit \
--constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
./scripts/ci/ci_run_static_checks.sh pylint-tests mypy yamllint flake8
./scripts/ci/static_checks/ci_run_static_checks.sh pylint-tests mypy yamllint flake8
static-checks-2:
timeout-minutes: 60
Expand All @@ -95,14 +95,14 @@ jobs:
path: ~/.cache/pre-commit
key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Static checks"
run: |
python -m pip install pre-commit \
--constraint requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt
./scripts/ci/ci_run_static_checks.sh
./scripts/ci/static_checks/ci_run_static_checks.sh
docs:
timeout-minutes: 60
Expand All @@ -113,9 +113,9 @@ jobs:
steps:
- uses: actions/checkout@master
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Build docs"
run: ./scripts/ci/ci_docs.sh
run: ./scripts/ci/docs/ci_docs.sh

build-prod-image:
timeout-minutes: 60
Expand All @@ -130,7 +130,7 @@ jobs:
steps:
- uses: actions/checkout@master
- name: "Build PROD image ${{ matrix.python-version }}"
run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh

prepare-backport-packages:
timeout-minutes: 60
Expand All @@ -148,12 +148,12 @@ jobs:
with:
python-version: '3.x'
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Prepare & test backport packages"
run: |
./scripts/ci/ci_prepare_and_test_backport_packages.sh
./scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh
trigger-tests:
timeout-minutes: 10
Expand All @@ -166,7 +166,7 @@ jobs:
- name: "Get count of changed python files"
run: |
set +e
./scripts/ci/ci_count_changed_files.sh ${GITHUB_SHA} \
./scripts/ci/tools/ci_count_changed_files.sh ${GITHUB_SHA} \
'^airflow|.github/workflows/|^Dockerfile|^scripts|^chart|^setup.py|^requirements|^tests|^kubernetes_tests'
echo "::set-output name=count::$?"
id: trigger-tests
Expand Down Expand Up @@ -209,15 +209,15 @@ jobs:
with:
python-version: '3.x'
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- uses: engineerd/[email protected]
name: Setup Kind Cluster
with:
version: "${{ matrix.kind-version }}"
name: airflow-python-${{matrix.python-version}}-${{matrix.kubernetes-version}}
config: "scripts/ci/kubernetes/kind-cluster-conf.yaml"
- name: "Deploy app to cluster"
run: ./scripts/ci/ci_deploy_app_to_kubernetes.sh
run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
- name: Cache virtualenv for kubernetes testing
uses: actions/cache@v2
env:
Expand All @@ -227,7 +227,7 @@ jobs:
key: "${{ env.cache-name }}-${{ github.job }}-\
${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt') }}"
- name: "Tests"
run: ./scripts/ci/ci_run_kubernetes_tests.sh
run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
- uses: actions/upload-artifact@v2
name: Upload KinD logs
# Always run this, even if one of th previous steps failed.
Expand Down Expand Up @@ -262,11 +262,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
with:
python-version: '3.x'
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests"
run: ./scripts/ci/ci_run_airflow_testing.sh
run: ./scripts/ci/testing/ci_run_airflow_testing.sh

tests-mysql:
timeout-minutes: 80
Expand Down Expand Up @@ -294,11 +294,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
with:
python-version: '3.x'
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests"
run: ./scripts/ci/ci_run_airflow_testing.sh
run: ./scripts/ci/testing/ci_run_airflow_testing.sh

tests-sqlite:
timeout-minutes: 80
Expand All @@ -324,11 +324,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
with:
python-version: '3.x'
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests"
run: ./scripts/ci/ci_run_airflow_testing.sh
run: ./scripts/ci/testing/ci_run_airflow_testing.sh

tests-quarantined:
timeout-minutes: 80
Expand Down Expand Up @@ -357,11 +357,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
with:
python-version: '3.x'
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests"
run: ./scripts/ci/ci_run_airflow_testing.sh
run: ./scripts/ci/testing/ci_run_airflow_testing.sh

helm-tests:
timeout-minutes: 5
Expand Down Expand Up @@ -391,11 +391,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
- uses: actions/checkout@master
- uses: actions/setup-python@v1
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Generate requirements"
run: ./scripts/ci/ci_generate_requirements.sh
run: ./scripts/ci/requirements/ci_generate_requirements.sh

push-prod-images-to-github-cache:
timeout-minutes: 80
Expand All @@ -419,11 +419,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
steps:
- uses: actions/checkout@master
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build PROD images ${{ matrix.python-version }}"
run: ./scripts/ci/ci_prepare_prod_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- name: "Push PROD images ${{ matrix.python-version }}"
run: ./scripts/ci/ci_push_production_images.sh
run: ./scripts/ci/images/ci_push_production_images.sh

push-ci-images-to-github-cache:
timeout-minutes: 40
Expand Down Expand Up @@ -451,11 +451,11 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
steps:
- uses: actions/checkout@master
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image"
run: ./scripts/ci/ci_prepare_ci_image_on_ci.sh
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Push CI image ${{ matrix.python-version }}"
run: ./scripts/ci/ci_push_ci_image.sh
run: ./scripts/ci/images/ci_push_ci_image.sh

tag-repo-nightly:
timeout-minutes: 60
Expand All @@ -473,7 +473,7 @@ ${{ hashFiles('requirements/requirements-python${{matrix.python-version}}.txt')
steps:
- uses: actions/checkout@master
- name: "Free space"
run: ./scripts/ci/ci_free_space_on_ci.sh
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Tag commit"
run: |
BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///')
Expand Down
36 changes: 18 additions & 18 deletions .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,43 +205,43 @@ repos:
- id: lint-dockerfile
name: Lint dockerfile
language: system
entry: "./scripts/ci/pre_commit_lint_dockerfile.sh"
entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh"
files: ^Dockerfile.*$
pass_filenames: true
- id: setup-order
name: Checks for an order of dependencies in setup.py
language: python
files: ^setup.py$
pass_filenames: false
entry: ./scripts/ci/pre_commit_check_order_setup.py
entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
- id: update-breeze-file
name: Update output of breeze command in BREEZE.rst
entry: "./scripts/ci/pre_commit_breeze_cmd_line.sh"
entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh"
language: system
files: ^BREEZE.rst$|^breeze$|^breeze-complete$
pass_filenames: false
- id: update-local-yml-file
name: Update mounts in the local yml file
entry: "./scripts/ci/pre_commit_local_yml_mounts.sh"
entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh"
language: system
files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml"
pass_filenames: false
- id: update-setup-cfg-file
name: Update setup.cfg file with all licenses
entry: "./scripts/ci/pre_commit_setup_cfg_file.sh"
entry: "./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh"
language: system
files: ^setup.cfg$
pass_filenames: false
- id: build-providers-dependencies
name: Build cross-dependencies for providers packages
entry: "./scripts/ci/pre_commit_build_providers_dependencies.sh"
entry: "./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh"
language: system
files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$
pass_filenames: false
- id: update-extras
name: Update extras in documentation
entry: "./scripts/ci/pre_commit_update_extras.sh"
language: system
entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py
language: python
files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$
pass_filenames: false
- id: pydevd
Expand Down Expand Up @@ -324,62 +324,62 @@ repos:
pass_filenames: true
- id: check-integrations
name: Check if integration list is aligned
entry: ./scripts/ci/pre_commit_check_integrations.sh
entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
language: system
pass_filenames: false
files: ^common/_common_values.sh$|^breeze-complete$
- id: build
name: Check if image build is needed
entry: ./scripts/ci/pre_commit_ci_build.sh 3.6 false
entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false
language: system
always_run: true
pass_filenames: false
- id: check-apache-license
name: Check if licenses are OK for Apache
entry: "./scripts/ci/pre_commit_check_license.sh"
entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh"
language: system
files: ^.*LICENSE.*$|^.*LICENCE.*$
pass_filenames: false
- id: airflow-config-yaml
name: Checks for consistency between config.yml and default_config.cfg
language: python
entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
files: "config.yml$|default_airflow.cfg$|default.cfg$"
pass_filenames: false
require_serial: false
entry: ./scripts/ci/pre_commit_yaml_to_cfg.py
require_serial: true
additional_dependencies: ['pyyaml']
- id: mypy
name: Run mypy
language: system
entry: "./scripts/ci/pre_commit_mypy.sh"
entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
files: \.py$
exclude: ^dev|^backport_packages
require_serial: true
- id: pylint
name: Run pylint for main sources
language: system
entry: "./scripts/ci/pre_commit_pylint_main.sh"
entry: "./scripts/ci/pre_commit/pre_commit_pylint_main.sh"
files: \.py$
exclude: ^tests/.*\.py$|^scripts/.*\.py$|^dev|^backport_packages|^kubernetes_tests
pass_filenames: true
require_serial: true # Pylint tests should be run in one chunk to detect all cycles
- id: pylint-tests
name: Run pylint for tests
language: system
entry: "./scripts/ci/pre_commit_pylint_tests.sh"
entry: "./scripts/ci/pre_commit/pre_commit_pylint_tests.sh"
files: ^tests/.*\.py$
pass_filenames: true
require_serial: true
- id: flake8
name: Run flake8
language: system
entry: "./scripts/ci/pre_commit_flake8.sh"
entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh"
files: \.py$
pass_filenames: true
- id: bat-tests
name: Run BATS bash tests for changed bash files
language: system
entry: "./scripts/ci/pre_commit_bat_tests.sh"
entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh"
files: ^breeze$|^breeze-complete$|\.sh$|\.bash$
pass_filenames: false
- id: stylelint
Expand Down
1 change: 1 addition & 0 deletions .rat-excludes
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,3 +91,4 @@ input_notebook.ipynb

# .git might be a file in case of worktree
.git
tmp
2 changes: 1 addition & 1 deletion BREEZE.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -922,7 +922,7 @@ by the root user, you can fix the ownership of those files by running this scrip

.. code-block::
./scripts/ci/ci_fix_ownership.sh
./scripts/ci/tools/ci_fix_ownership.sh
Mounting Local Sources to Breeze
--------------------------------
Expand Down
Loading

0 comments on commit faec41e

Please sign in to comment.