diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000000..592000cdccc0df --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [commit, push] +repos: + - repo: meta + hooks: + - id: check-hooks-apply + - repo: local + hooks: + - id: lint-dockerfile + name: Lint dockerfile + language: system + entry: ./scripts/ci/ci_lint_dockerfile.sh + files: ^Dockerfile.*$ + - id: mypy + name: Run mypy + language: system + entry: ./scripts/ci/ci_mypy.sh + files: \.py$ + require_serial: true + - id: flake8 + name: Run flake8 + language: system + entry: ./scripts/ci/ci_flake8.sh + files: \.py$ + require_serial: true diff --git a/.travis.yml b/.travis.yml index 6ce0756ced526d..93840223e401fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,8 @@ language: python env: global: - BUILD_ID=${TRAVIS_BUILD_ID} + - AIRFLOW_CONTAINER_MOUNT_HOST_VOLUMES="false" + - AIRFLOW_CI_SILENT="false" - CI="true" python: "3.6" stages: @@ -36,9 +38,9 @@ jobs: env: BACKEND=sqlite ENV=docker python: "3.5" stage: test - - name: "Tests mysql python 2.7" + - name: "Tests mysql python 3.6" env: BACKEND=mysql ENV=docker - python: "2.7" + python: "3.6" stage: test - name: "Tests postgres kubernetes python 3.6 (persistent)" env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.13.0 KUBERNETES_MODE=persistent_mode @@ -56,20 +58,11 @@ jobs: env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.9.0 KUBERNETES_MODE=git_mode python: "2.7" stage: test - - name: Flake8 - stage: pre-test - script: ./scripts/ci/ci_flake8.sh - - name: mypy - stage: pre-test - script: ./scripts/ci/ci_mypy.sh - - name: Check license header + - name: "Static checks" stage: pre-test - script: ./scripts/ci/ci_check_license.sh - - name: Lint Dockerfile - stage: pre-test - script: ./scripts/ci/ci_lint_dockerfile.sh + script: ./scripts/ci/ci_run_all_static_tests.sh - name: Check docs - stage: test + stage: pre-test script: ./scripts/ci/ci_docs.sh services: - docker diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b6c6dad4ad7c5e..7ff0f0f15b8fa4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,11 +42,12 @@ little bit helps, and credit will always be given. - [Using the Docker Compose environment](#using-the-docker-compose-environment) - [Entering bash shell in Docker Compose environment](#entering-bash-shell-in-docker-compose-environment) - [Running individual tests within the container](#running-individual-tests-within-the-container) - - [Running static code analysis](#running-static-code-analysis) + - [Running static code analysis](#running-static-code-analysis) - [Running static code analysis from the host](#running-static-code-analysis-from-the-host) - [Running static code analysis in the docker compose environment](#running-static-code-analysis-in-the-docker-compose-environment) - [Running static code analysis on selected files/modules](#running-static-code-analysis-on-selected-filesmodules) - [Docker images](#docker-images) + - [Default behaviour for user interaction](#default-behaviour-for-user-interaction) - [Local Docker Compose scripts](#local-docker-compose-scripts) - [Running the whole suite of tests](#running-the-whole-suite-of-tests) - [Stopping the environment](#stopping-the-environment) @@ -55,7 +56,14 @@ little bit helps, and credit will always be given. - [Force pulling the images](#force-pulling-the-images) - [Cleaning up cached Docker images/containers](#cleaning-up-cached-docker-imagescontainers) - [Troubleshooting](#troubleshooting) -- [Git hooks](#git-hooks) +- [Pylint checks](#pylint-checks) +- [Pre-commit hooks](#pre-commit-hooks) + - [Installing pre-commit hooks](#installing-pre-commit-hooks) + - [Docker images for pre-commit hooks](#docker-images-for-pre-commit-hooks) + - [Pre-commit hooks installed](#pre-commit-hooks-installed) + - [Using pre-commit hooks](#using-pre-commit-hooks) + - [Skipping pre-commit hooks](#skipping-pre-commit-hooks) + - [Advanced pre-commit usage](#advanced-pre-commit-usage) - [Pull Request Guidelines](#pull-request-guidelines) - [Testing on Travis CI](#testing-on-travis-ci) - [Travis CI GitHub App (new version)](#travis-ci-github-app-new-version) @@ -141,6 +149,9 @@ environment and you can easily debug the code locally. You can also have access contains all the necessary requirements and use it in your local IDE - this aids autocompletion, and running tests directly from within the IDE. +It is **STRONGLY** encouraged to also install and use [Pre commit hooks](#pre-commit-hooks) for your local +development environment. They will speed up your development cycle speed a lot. + The disadvantage is that you have to maintain your dependencies and local environment consistent with other development environments that you have on your local machine. @@ -162,7 +173,7 @@ managers like yum, apt-get for Linux, or Homebrew for Mac OS at first. Refer to the [Dockerfile](Dockerfile) for a comprehensive list of required packages. In order to use your IDE you need you can use the virtual environment. Ideally -you should setup virtualenv for all python versions that Airflow supports (2.7, 3.5, 3.6). +you should setup virtualenv for all python versions that Airflow supports (3.5, 3.6). An easy way to create the virtualenv is to use [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/) - it allows you to easily switch between virtualenvs using `workon` command and mange @@ -173,7 +184,23 @@ mkvirtualenv --python=python ``` Then you need to install python PIP requirements. Typically it can be done with: -`pip install -e ".[devel]"`. Then you need to run `airflow db init` to create sqlite database. +`pip install -e ".[devel]"`. + +Note - if you have trouble installing mysql client on MacOS and you have an error similar to +``` +ld: library not found for -lssl +``` + +you should set LIBRARY_PATH before running `pip install`: + +``` +export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/opt/openssl/lib/ +``` + +After creating the virtualenv, run this command to create the Airflow sqlite database: +``` +airflow db init +``` Once initialization is done, you should select the virtualenv you initialized as the project's default virtualenv in your IDE and run tests efficiently. @@ -295,12 +322,6 @@ If you use zsh, you should run this command: echo 'export PATH="/usr/local/opt/gnu-getopt/bin:$PATH"' >> ~/.zprofile . ~/.zprofile -If you use zsh, you should run this command: - -```bash -echo 'export PATH="/usr/local/opt/gnu-getopt/bin:$PATH"' >> ~/.zprofile -. ~/.zprofile -``` * Login and logout afterwards If you are on Linux: @@ -327,7 +348,7 @@ PYTHON_VERSION=3.5 BACKEND=postgres ENV=docker ./scripts/ci/local_ci_enter_envir Once you are inside the environment you can run individual tests as described in [Running individual tests](#running-individual-tests). -### Running static code analysis +## Running static code analysis We have a number of static code checks that are run in Travis CI but you can run them locally as well. All the scripts are available in [scripts/ci](scripts/ci) folder. @@ -344,19 +365,20 @@ they should give the same results as the tests run in TravisCI without special e You can trigger the static checks from the host environment, without entering Docker container. You do that by running appropriate scripts (The same is done in TravisCI) -* [ci_docs.sh](scripts/ci/ci_docs.sh) - checks that documentation can be built without warnings. -* [ci_flake8.sh](scripts/ci/ci_flake8.sh) - runs flake8 source code style guide enforcement tool -* [ci_mypy.sh](scripts/ci/ci_mypy.sh) - runs mypy type annotation consistency check -* [ci_lint_dockerfile.sh](scripts/ci/ci_lint_dockerfile.sh) - runs lint checker for the Dockerfile -* [ci_check_license.sh](scripts/ci/ci_check_license.sh) - checks if all licences are present in the sources +* [scripts/ci/ci_check_license.sh](scripts/ci/ci_check_license.sh) - checks if all licences are present in the sources +* [scripts/ci/ci_docs.sh](scripts/ci/ci_docs.sh) - checks that documentation can be built without warnings. +* [scripts/ci/ci_flake8.sh](scripts/ci/ci_flake8.sh) - runs flake8 source code style guide enforcement tool +* [scripts/ci/ci_lint_dockerfile.sh](scripts/ci/ci_lint_dockerfile.sh) - runs lint checker for the Dockerfile +* [scripts/ci/ci_mypy.sh](scripts/ci/ci_mypy.sh) - runs mypy type annotation consistency check -Those scripts are optimised for time of rebuilds of docker image. The image will be automatically -rebuilt when needed (for example when dependencies change). +The scripts will fail by default when image rebuild is needed (for example when dependencies change) +and provide instruction on how to rebuild the images. You can control the default behaviour as explained in +[Default behaviour for user interaction](#default-behaviour-for-user-interaction) -You can also force rebuilding of the image by deleting [.build](./build) -directory which keeps cached information about the images built. +You can force rebuilding of the images by deleting [.build](./build) directory. This directory keeps cached +information about the images already built and you can safely delete it if you want to start from the scratch. -Documentation after it is built, is available in [docs/_build/html](docs/_build/html) folder. +After Documentation is built, the html results are available in [docs/_build/html](docs/_build/html) folder. This folder is mounted from the host so you can access those files in your host as well. #### Running static code analysis in the docker compose environment @@ -394,35 +416,67 @@ And similarly for other scripts. ## Docker images - -For all development related tasks related to integration tests and static code checks we are using Docker +For all development tasks related integration tests and static code checks we are using Docker images that are maintained in Dockerhub under `apache/airflow` repository. -There are two images that we currently manage: +There are three images that we currently manage: -* Slim CI image that is used for static code checks (size around 500MB) - labelled following the pattern - of -python-ci-slim (for example master-python3.6-ci-slim) +* Slim CI image that is used for static code checks (size around 500MB) - tag follows the pattern + of `-python-ci-slim` (for example `master-python3.6-ci-slim`). The image is built + using the [Dockerfile](Dockerfile) dockerfile. * Full CI image that is used for testing - containing a lot more test-related installed software - (size around 1GB) - labelled following the pattern of -python-ci - (for example master-python3.6-ci) - -When you run tests or enter environment or run local static checks, the first time you do it, -the necessary local images will be pulled and built for you automatically from DockerHub. Then -the scripts will check automatically if the image needs to be re-built if needed and will do that -automatically for you. + (size around 1GB) - tag follows the pattern of `-python-ci` + (for example `master-python3.6-ci`). The image is built using the [Dockerfile](Dockerfile) dockerfile. +* Checklicence image - an image that is used during licence check using Apache RAT tool. It does not + require any of the dependencies that the two CI images need so it is built using different Dockerfile + [Dockerfile-checklicence](Dockerfile-checklicence) and only contains Java + Apache RAT tool. The image is + labeled with `checklicence` image. + +We also use a very small [Dockerfile-context](Dockerfile-context) dockerfile in order to fix file permissions +for an obscure permission problem with Docker caching but it is not stored in `apache/airflow` registry. + +Before you run tests or enter environment or run local static checks, the necessary local images should be +pulled and built from DockerHub. This happens automatically for the test environment but you need to +manually trigger it for static checks as described in +[Building the images](#building-the-images) and +[Force pulling and building the images](#force-pulling-the-images)). The static checks will fail and inform +what to do if the image is not yet built. Note that building image first time pulls the pre-built version of images from Dockerhub might take a bit of time - but this wait-time will not repeat for any subsequent source code change. However, changes to sensitive files like setup.py or Dockerfile will trigger a rebuild that might take more time (but it is highly optimised to only rebuild what's needed) -You can also [Build the images](#building-the-images) or -[Force pull and build the images](#force-pulling-the-images)) manually at any time. +In most cases re-building an image requires connectivity to network (for example to download new +dependencies). In case you work offline and do not want to rebuild the images when needed - you might set +`ASSUME_NO_TO_ALL_QUESTIONS` variable to `true` as described in the +[Default behaviour for user interaction](#default-behaviour-for-user-interaction) chapter. See [Troubleshooting section](#troubleshooting) for steps you can make to clean the environment. -Once you performed the first build, the images are rebuilt locally rather than pulled - unless you -force pull the images. But you can force it using the scripts described below. +## Default behaviour for user interaction + +Sometimes during the build user is asked whether to perform an action, skip it, or quit. This happens in case +of image rebuilding and image removal - they can take a lot of time and they are potentially destructive. +For automation scripts, you can export one of the three variables to control the default behaviour. +``` +export ASSUME_YES_TO_ALL_QUESTIONS="true" +``` +If `ASSUME_YES_TO_ALL_QUESTIONS` is set to `true`, the images will automatically rebuild when needed. +Images are deleted without asking. + +``` +export ASSUME_NO_TO_ALL_QUESTIONS="true" +``` +If `ASSUME_NO_TO_ALL_QUESTIONS` is set to `true`, the old images are used even if re-building is needed. +This is useful when you work offline. Deleting images is aborted. + +``` +export ASSUME_QUIT_TO_ALL_QUESTIONS="true" +``` +If `ASSUME_QUIT_TO_ALL_QUESTIONS` is set to `true`, the whole script is aborted. Deleting images is aborted. + +If more than one variable is set, YES takes precedence over NO which take precedence over QUIT. ## Local Docker Compose scripts @@ -480,7 +534,6 @@ Docker-compose environment starts a number of docker containers and keep them ru You can tear them down by running [/scripts/ci/local_ci_stop_environment.sh](scripts/ci/local_ci_stop_environment.sh) - ### Fixing file/directory ownership On Linux there is a problem with propagating ownership of created files (known Docker problem). Basically @@ -495,6 +548,12 @@ you can fix the ownership of those files by running You can manually trigger building of the local images using [scripts/ci/local_ci_build.sh](scripts/ci/local_ci_build.sh). +The scripts that build the images are optimised to minimise the time needed to rebuild the image when +the source code of Airflow evolves. This means that if you already had the image locally downloaded and built, +the scripts will determine, the rebuild is needed in the first place. Then it will make sure that minimal +number of steps are executed to rebuild the parts of image (for example PIP dependencies) that will give +you an image consistent with the one used during Continuous Integration. + ### Force pulling the images You can also force-pull the images before building them locally so that you are sure that you download @@ -521,7 +580,7 @@ available for Docker. See [Docker for Mac - Space](https://docs.docker.com/docke ## Troubleshooting -In case you have problems with the Docker Compose environment - try the following (after each step you +If you are having problems with the Docker Compose environment - try the following (after each step you can check if your problem is fixed) 1. Check if you have [enough disk space](#prerequisites) in Docker if you are on MacOS. @@ -531,73 +590,104 @@ can check if your problem is fixed) 5. [Fix file/directory ownership](#fixing-filedirectory-ownership) 6. Restart your docker engine and try again 7. Restart your machine and try again -1. Remove and re-install Docker CE, then start with [force pulling the images](#force-pulling-the-images) +8. Remove and re-install Docker CE, then start with [force pulling the images](#force-pulling-the-images) In case the problems are not solved, you can set VERBOSE variable to "true" (`export VERBOSE="true"`) and rerun failing command, and copy & paste the output from your terminal, describe the problem and post it in [Airflow Slack](https://apache-airflow-slack.herokuapp.com/) #troubleshooting channel. -# Git hooks +# Pre-commit hooks + +Pre-commit hooks are fantastic way of speeding up your local development cycle. Those pre-commit checks will +only check the files that you are currently working on which make them fast. Yet they are using exactly +the same environment as the CI checks are using, so you can be pretty sure your modifications +will be ok for CI if they pass pre-commit checks. + +You are *STRONGLY* encouraged to install pre-commit hooks as they speed up your development and place less +burden on the CI infrastructure. + +We have integrated the fantastic [pre-commit](https://pre-commit.com/) framework in our development workflow. +You need to have python 3.6 installed in your host in order to install and use it. It's best to run your +commits when you have your local virtualenv for Airflow activated (then pre-commit and other +dependencies are automatically installed). You can also install pre-commit manually using `pip install`. + +The pre-commit hooks require Docker Engine to be configured as the static checks static checks are +executed in docker environment. You should build the images locally before installing pre-commit checks as +described in [Building the images](#building-the-images). In case you do not have your local images built +the pre-commit hooks fail and provide instructions on what needs to be done. + +## Installing pre-commit hooks + +``` +pre-commit install +``` + +Running the command by default turns on pre-commit checks for `commit` operations in git. -Another great way of automating linting and testing is to use - [Git Hooks](https://git-scm.com/book/uz/v2/Customizing-Git-Git-Hooks). For example you could create a -`pre-commit` file based on the Travis CI Pipeline so that before each commit a local pipeline will be -triggered and if this pipeline fails (returns an exit code other than `0`) the commit does not come through. -This "in theory" has the advantage that you can not commit any code that fails that again reduces the -errors in the Travis CI Pipelines. +You can also decide to install the checks also for `pre-push` operation: -Since there are a lot of tests the script would last very long so you probably only should test your - new -feature locally. +``` +pre-commit install -t pre-push +``` -The following example of a `pre-commit` file allows you.. -- to lint your code via flake8 -- to test your code via nosetests in a docker container based on python 2 -- to test your code via nosetests in a docker container based on python 3 +You can see advanced usage of the install method via ``` -#!/bin/sh +pre-commit install --help +``` -GREEN='\033[0;32m' -NO_COLOR='\033[0m' +## Docker images for pre-commit hooks -setup_python_env() { - local venv_path=${1} +Before running the pre-commit hooks you must first build the docker images locally as described in +[Building the images](#building-the-images) chapter. - echo -e "${GREEN}Activating python virtual environment ${venv_path}..${NO_COLOR}" - source ${venv_path} -} -run_linting() { - local project_dir=$(git rev-parse --show-toplevel) +Sometimes your image is outdated (when dependencies change) and needs to be rebuilt because some +dependencies have been changed. In such case the docker build pre-commit will fail and inform +you that you should rebuild the image with REBUILD="true" environment variable set. - echo -e "${GREEN}Running flake8 over directory ${project_dir}..${NO_COLOR}" - flake8 ${project_dir} -} -run_testing_in_docker() { - local feature_path=${1} - local airflow_py2_container=${2} - local airflow_py3_container=${3} - echo -e "${GREEN}Running tests in ${feature_path} in airflow python 2 docker container..${NO_COLOR}" - docker exec -i -w /airflow/ ${airflow_py2_container} nosetests -v ${feature_path} - echo -e "${GREEN}Running tests in ${feature_path} in airflow python 3 docker container..${NO_COLOR}" - docker exec -i -w /airflow/ ${airflow_py3_container} nosetests -v ${feature_path} -} +## Pre-commit hooks installed -set -e -# NOTE: Before running this make sure you have set the function arguments correctly. -setup_python_env /Users/feluelle/venv/bin/activate -run_linting -run_testing_in_docker tests/contrib/hooks/test_imap_hook.py dazzling_chatterjee quirky_stallman +In airflow we have the following checks: +```text +lint-dockerfile Lint dockerfile +mypy Run mypy +pylint Run pylint +flake8 Run flake8 ``` +## Using pre-commit hooks + +After installing pre-commit hooks are run automatically when you commit the code, but you can +run pre-commit hooks manually as needed. + +*You can run all checks on your staged files by running:* +`pre-commit run` + +*You can run only one mypy check on your staged files by running:* +`pre-commit run mypy` + +*You can run only one mypy checks manually on all files by running:* +`pre-commit run mypy --all-files` + +*You can run all checks manually on all files by running:* +`SKIP=pylint pre-commit run --all-files` + +Note this might be very slow for individual tests with pylint because of passing individual files. It is +recommended to run `/scripts/ci/ci_pylint_main.sh` (for the main application files) or +`/scripts/ci/ci_pylint_tests.sh` (for tests) for pylint check. +You can also adding SKIP=pylint variable (as in the example above) if you run pre-commit hooks with --all-files switch. + +*You can skip one or more of the checks by specifying comma-separated list of checks to skip in SKIP variable:* +`SKIP=pylint,mypy pre-commit run --all-files` + +## Skipping pre-commit hooks -For more information on how to run a subset of the tests, take a look at the -nosetests docs. +You can always skip running the tests by providing `--no-verify` flag to `git commit` command. -See also the list of test classes and methods in `tests/core.py`. +## Advanced pre-commit usage -Feel free to customize based on the extras available in [setup.py](./setup.py) +You can check other usages of pre-commit framework at [Pre-commit website](https://pre-commit.com/) # Pull Request Guidelines @@ -624,8 +714,8 @@ are often sufficient. Make sure to follow the Sphinx compatible standards. 1. The pull request should work for Python 3.5 and 3.6. 1. As Airflow grows as a project, we try to enforce a more consistent style and try to follow the Python community guidelines. We currently enforce most [PEP8](https://www.python.org/dev/peps/pep-0008/) and a -few other linting rules - described in [Running linting and tests](#running-linting-and-tests). It's a good -idea to run tests locally before opening PR. +few other linting rules - described in [Running static code analysis locally](#running-static-code-analysis-locally). +It's a good idea to run tests locally before opening PR. 1. Please read this excellent [article](http://chris.beams.io/posts/git-commit/) on commit messages and adhere to them. It makes the lives of those who come after you a lot easier. diff --git a/airflow/contrib/hooks/bigquery_hook.py b/airflow/contrib/hooks/bigquery_hook.py index 9affa20b7427c7..edf4eab528e735 100644 --- a/airflow/contrib/hooks/bigquery_hook.py +++ b/airflow/contrib/hooks/bigquery_hook.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # pylint: disable=too-many-lines # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/confirm b/confirm index 0580e406b72ea9..4e8f1884000f6f 100755 --- a/confirm +++ b/confirm @@ -17,15 +17,27 @@ # under the License. set -euo pipefail -if [[ "${ASSUME_YES:=false}" == "true" ]]; then +if [[ "${ASSUME_YES_TO_ALL_QUESTIONS:=false}" == "true" ]]; then exit 0 fi -read -r -p "${1}. Are you sure? [y/N] " response +if [[ "${ASSUME_NO_TO_ALL_QUESTIONS:=false}" == "true" ]]; then + exit 1 +fi + +if [[ "${ASSUME_QUIT_TO_ALL_QUESTIONS:=false}" == "true" ]]; then + exit 2 +fi + + +read -r -p "${1}. Are you sure? [y/N/q] " response case "$response" in [yY][eE][sS]|[yY]) exit 0 ;; + [qQ][uU][iI|[tT]|[qQ]) + exit 2 + ;; *) exit 1 ;; diff --git a/docs/conf.py b/docs/conf.py index 6d22b6b245a06a..a58d5ba27633cf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,8 +31,10 @@ # # All configuration values have a default; values that are commented out # serve to show the default. +"""Configuration of Airflow Docs""" import os import sys +from typing import Dict import airflow @@ -255,7 +257,7 @@ # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] -import sphinx_rtd_theme +import sphinx_rtd_theme # pylint: disable=wrong-import-position,wrong-import-order html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] @@ -340,7 +342,7 @@ # Additional stuff for the LaTeX preamble. # 'preamble': '', -} +} # type: Dict[str,str] # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, diff --git a/hooks/build b/hooks/build index a733d70f3cee1c..0c3c8f21e8fe8f 100755 --- a/hooks/build +++ b/hooks/build @@ -147,15 +147,19 @@ export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} # own docker images. In this case you can build images locally and push them export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} +# You can override python binary with specific version +export PYTHON_BINARY=${PYTHON_BINARY:=python} + # Determine python version from the local python on the path. # You can override it with PYTHON_VERSION because all python version dependencies # Are determined in Docker images. If you pass IMAGE_NAME, python version will be extracted from the name # This default Python version is later overridden in case IMAGE_NAME is passed export PYTHON_VERSION=\ -${PYTHON_VERSION:=$(python -c 'import sys;print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')} +${PYTHON_VERSION:=$(${PYTHON_BINARY} -c \ + 'import sys;print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')} # shellcheck source=./_default_branch.sh -. ${MY_DIR}/_default_branch.sh +. "${MY_DIR}/_default_branch.sh" # Source brnnch will be set in DockerHub SOURCE_BRANCH=${SOURCE_BRANCH:=${DEFAULT_BRANCH}} @@ -259,7 +263,7 @@ AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE=${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_ pwd # Determine version of the Airflow from version.py -AIRFLOW_VERSION=$(cat airflow/version.py - << EOF | python +AIRFLOW_VERSION=$(cat airflow/version.py - << EOF | ${PYTHON_BINARY} print(version.replace("+","")) EOF ) @@ -268,7 +272,7 @@ export AIRFLOW_VERSION # Check if we are running in the CI environment CI=${CI:="false"} -if [[ ${CI} == "true" ]]; then +if [[ "${CI}" == "true" ]]; then NON_CI="false" else NON_CI="true" @@ -425,57 +429,65 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CHECKLICENCE_IMAGE}" fi - for IMAGE in ${IMAGES_TO_PULL} - do + DOCKER_CACHE_DIRECTIVE_CI=() + DOCKER_CACHE_DIRECTIVE_CI_SLIM=() + if [[ ${IMAGES_TO_PULL} == "" ]]; then echo - echo "Checking whether image ${IMAGE} needs to be pulled." + echo "Skipping building of all images." echo - PULL_IMAGE="false" - if [[ "${AIRFLOW_CONTAINER_FORCE_PULL_IMAGES}" == "true" ]]; then + else + for IMAGE in ${IMAGES_TO_PULL} + do echo - echo "Pulling images is forced. Pulling ${IMAGE}" + echo "Checking whether image ${IMAGE} needs to be pulled." echo - PULL_IMAGE="true" - else - IMAGE_HASH=$(docker images -q "${IMAGE}" 2> /dev/null) - if [[ "${IMAGE_HASH}" == "" ]]; then + PULL_IMAGE="false" + if [[ "${AIRFLOW_CONTAINER_FORCE_PULL_IMAGES}" == "true" ]]; then echo - echo "No image ${IMAGE} locally available. Pulling for the first time." + echo "Pulling images is forced. Pulling ${IMAGE}" echo PULL_IMAGE="true" else - echo - echo "Image ${IMAGE} is in local registry (${IMAGE_HASH}). Not pulling it!" - echo - PULL_IMAGE="false" + IMAGE_HASH=$(docker images -q "${IMAGE}" 2> /dev/null) + if [[ "${IMAGE_HASH}" == "" ]]; then + echo + echo "No image ${IMAGE} locally available. Pulling for the first time." + echo + PULL_IMAGE="true" + else + echo + echo "Image ${IMAGE} is in local registry (${IMAGE_HASH}). Not pulling it!" + echo + PULL_IMAGE="false" + fi fi - fi - if [[ "${PULL_IMAGE}" == "true" ]]; then - if [[ "${AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES}" == "true" ]]; then - echo - echo "Skipping pulling image ${IMAGE}" - echo + if [[ "${PULL_IMAGE}" == "true" ]]; then + if [[ "${AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES}" == "true" ]]; then + echo + echo "Skipping pulling image ${IMAGE}" + echo + else + echo + set -x + docker pull "${IMAGE}" || true + set +x + echo + fi + fi + if [[ "${IMAGE}" == "${AIRFLOW_CI_IMAGE}" ]]; then + DOCKER_CACHE_DIRECTIVE_CI+=("--cache-from" "${IMAGE}") + elif [[ "${IMAGE}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then + DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--cache-from" "${IMAGE}") + elif [[ "${IMAGE}" == "${AIRFLOW_CHECKLICENCE_IMAGE}" ]]; then + DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--cache-from" "${IMAGE}") else echo - set -x - docker pull "${IMAGE}" || true - set +x + echo "Don't know how to set cache directive for ${IMAGE}. Exiting" echo + exit 1 fi - fi - if [[ "${IMAGE}" == "${AIRFLOW_CI_IMAGE}" ]]; then - DOCKER_CACHE_DIRECTIVE_CI+=("--cache-from" "${IMAGE}") - elif [[ "${IMAGE}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then - DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--cache-from" "${IMAGE}") - elif [[ "${IMAGE}" == "${AIRFLOW_CHECKLICENCE_IMAGE}" ]]; then - DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--cache-from" "${IMAGE}") - else - echo - echo "Don't know how to set cache directive for ${IMAGE}. Exiting" - echo - exit 1 - fi - done + done + fi fi start_step "Setting cache options" @@ -600,13 +612,16 @@ else if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_DEFAULT}" add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_DEFAULT}" + save_to_file AIRFLOW_SLIM_CI_IMAGE_DEFAULT fi if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_LATEST}" add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_LATEST}" + save_to_file AIRFLOW_SLIM_CI_IMAGE_LATEST if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT}" add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT}" + save_to_file AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT fi fi fi @@ -627,13 +642,16 @@ else if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_IMAGE_DEFAULT}" add_image_to_push "${AIRFLOW_CI_IMAGE_DEFAULT}" + save_to_file AIRFLOW_CI_IMAGE_DEFAULT fi if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_IMAGE_LATEST}" add_image_to_push "${AIRFLOW_CI_IMAGE_LATEST}" + save_to_file AIRFLOW_CI_IMAGE_LATEST if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_IMAGE_LATEST_DEFAULT}" add_image_to_push "${AIRFLOW_CI_IMAGE_LATEST_DEFAULT}" + save_to_file AIRFLOW_CI_IMAGE_LATEST_DEFAULT fi fi fi diff --git a/hooks/push b/hooks/push index eae33aca2988df..91cf0969ea6f34 100755 --- a/hooks/push +++ b/hooks/push @@ -1,25 +1,20 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This is hook build used by DockerHub. We are also using it -# on Travis CI to potentially rebuild (and refresh layers that -# are not cached) Docker images that are used to run CI jobs +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # We do not push in the push step because we are building multiple images in the build step # and it is difficult to pass list of the built images from the build to push phase diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index bc40722ba22e52..d2fd5b0de2f917 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -36,9 +36,18 @@ mkdir -p "${AIRFLOW_SOURCES}/.mypy_cache" mkdir -p "${AIRFLOW_SOURCES}/logs" mkdir -p "${AIRFLOW_SOURCES}/tmp" +# Dockerhub user and repo where the images are stored. +export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} +export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} +# Port on which webserver is exposed in host environment +export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="8080"} + +# Do not push images from here by default (push them directly from the build script on Dockerhub) +export AIRFLOW_CONTAINER_PUSH_IMAGES=${AIRFLOW_CONTAINER_PUSH_IMAGES:="false"} + # Disable writing .pyc files - slightly slower imports but not messing around when switching # Python version and avoids problems with root-owned .pyc files in host -export PYTHONDONTWRITEBYTECODE="true" +export PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE:="true"} # Read default branch name # shellcheck source=../../hooks/_default_branch.sh @@ -55,12 +64,24 @@ export AIRFLOW_CONTAINER_BRANCH_NAME=${AIRFLOW_CONTAINER_BRANCH_NAME:=${DEFAULT_ # AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS=${AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS:="true"} +function print_info() { + if [[ ${AIRFLOW_CI_SILENT:="false"} != "true" || ${AIRFLOW_CI_VERBOSE:="false"} == "true" ]]; then + echo "$@" + fi +} + +if [[ ${REBUILD:=false} == "true" ]]; then + print_info + print_info "Rebuilding is enabled. Assuming yes to all questions" + print_info + export ASSUME_YES_TO_ALL_QUESTIONS="true" +fi declare -a AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS if [[ ${AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS} == "true" ]]; then - echo - echo "Mounting host volumes to Docker" - echo + print_info + print_info "Mounting host volumes to Docker" + print_info AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS=( \ "-v" "${AIRFLOW_SOURCES}/airflow:/opt/airflow/airflow:cached" \ "-v" "${AIRFLOW_SOURCES}/.mypy_cache:/opt/airflow/.mypy_cache:cached" \ @@ -76,14 +97,14 @@ if [[ ${AIRFLOW_MOUNT_HOST_VOLUMES_FOR_STATIC_CHECKS} == "true" ]]; then "-v" "${AIRFLOW_SOURCES}/logs:/opt/airflow/logs:cached" \ "-v" "${AIRFLOW_SOURCES}/logs:/root/logs:cached" \ "-v" "${AIRFLOW_SOURCES}/tmp:/opt/airflow/tmp:cached" \ - "-e" "PYTHONDONTWRITEBYTECODE=true" \ + "--env" "PYTHONDONTWRITEBYTECODE" \ ) else - echo - echo "Skip mounting host volumes to Docker" - echo + print_info + print_info "Skip mounting host volumes to Docker" + print_info AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS=( \ - "-e" "PYTHONDONTWRITEBYTECODE=true" \ + "--env" "PYTHONDONTWRITEBYTECODE" \ ) fi @@ -120,22 +141,22 @@ function create_cache_directory() { function check_file_md5sum { local FILE="${1}" local MD5SUM - mkdir -pv "${BUILD_CACHE_DIR}/${THE_IMAGE}" + mkdir -pv "${BUILD_CACHE_DIR}/${THE_IMAGE_TYPE}" MD5SUM=$(md5sum "${FILE}") local MD5SUM_FILE - MD5SUM_FILE=${BUILD_CACHE_DIR}/${THE_IMAGE}/$(basename "${FILE}").md5sum + MD5SUM_FILE=${BUILD_CACHE_DIR}/${THE_IMAGE_TYPE}/$(basename "${FILE}").md5sum local MD5SUM_FILE_NEW MD5SUM_FILE_NEW=${CACHE_TMP_FILE_DIR}/$(basename "${FILE}").md5sum.new echo "${MD5SUM}" > "${MD5SUM_FILE_NEW}" local RET_CODE=0 if [[ ! -f "${MD5SUM_FILE}" ]]; then - echo "Missing md5sum for ${FILE}" + print_info "Missing md5sum for ${FILE}" RET_CODE=1 else diff "${MD5SUM_FILE_NEW}" "${MD5SUM_FILE}" >/dev/null RES=$? if [[ "${RES}" != "0" ]]; then - echo "The md5sum changed for ${FILE}" + print_info "The md5sum changed for ${FILE}" RET_CODE=1 fi fi @@ -149,13 +170,13 @@ function check_file_md5sum { function move_file_md5sum { local FILE="${1}" local MD5SUM_FILE - mkdir -pv "${BUILD_CACHE_DIR}/${THE_IMAGE}" - MD5SUM_FILE=${BUILD_CACHE_DIR}/${THE_IMAGE}/$(basename "${FILE}").md5sum + mkdir -pv "${BUILD_CACHE_DIR}/${THE_IMAGE_TYPE}" + MD5SUM_FILE=${BUILD_CACHE_DIR}/${THE_IMAGE_TYPE}/$(basename "${FILE}").md5sum local MD5SUM_FILE_NEW MD5SUM_FILE_NEW=${CACHE_TMP_FILE_DIR}/$(basename "${FILE}").md5sum.new if [[ -f "${MD5SUM_FILE_NEW}" ]]; then mv "${MD5SUM_FILE_NEW}" "${MD5SUM_FILE}" - echo "Updated md5sum file ${MD5SUM_FILE} for ${FILE}." + print_info "Updated md5sum file ${MD5SUM_FILE} for ${FILE}." fi } @@ -165,9 +186,9 @@ function move_file_md5sum { # it from the local docker cache rather than pull (unless forced) # function update_all_md5_files() { - echo - echo "Updating md5sum files" - echo + print_info + print_info "Updating md5sum files" + print_info for FILE in ${FILES_FOR_REBUILD_CHECK} do move_file_md5sum "${AIRFLOW_SOURCES}/${FILE}" @@ -176,7 +197,7 @@ function update_all_md5_files() { if [[ -n ${PYTHON_VERSION:=""} ]]; then SUFFIX="_${PYTHON_VERSION}" fi - touch "${BUILD_CACHE_DIR}/.built_${THE_IMAGE}${SUFFIX}" + touch "${BUILD_CACHE_DIR}/.built_${THE_IMAGE_TYPE}${SUFFIX}" } # @@ -203,7 +224,6 @@ function update_all_md5_files() { # function check_if_docker_build_is_needed() { set +e - for FILE in ${FILES_FOR_REBUILD_CHECK} do if ! check_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"; then @@ -238,36 +258,36 @@ function check_if_coreutils_installed() { #################### Parsing options/arguments if [[ ${GETOPT_RETVAL} != 4 || "${STAT_PRESENT}" != "0" || "${MD5SUM_PRESENT}" != "0" ]]; then - echo + print_info if [[ $(uname -s) == 'Darwin' ]] ; then echo >&2 "You are running ${CMDNAME} in OSX environment" echo >&2 "And you need to install gnu commands" echo >&2 echo >&2 "Run 'brew install gnu-getopt coreutils'" echo >&2 - echo >&2 "Then link the gnu-getopt to become default as suggested by brew by typing:" + echo >&2 "Then link the gnu-getopt to become default as suggested by brew." + echo >&2 + echo >&2 "If you use bash, you should run this command:" + echo >&2 echo >&2 "echo 'export PATH=\"/usr/local/opt/gnu-getopt/bin:\$PATH\"' >> ~/.bash_profile" echo >&2 ". ~/.bash_profile" echo >&2 - echo >&2 "if you use bash, or" + echo >&2 "If you use zsh, you should run this command:" echo >&2 - echo >&2 "echo 'export PATH=\"/usr/local/opt/gnu-getopt/bin:\$PATH\"' >> ~/.bash_profile" + echo >&2 "echo 'export PATH=\"/usr/local/opt/gnu-getopt/bin:\$PATH\"' >> ~/.zprofile" echo >&2 ". ~/.zprofile" echo >&2 - echo >&2 "if you use zsh" - echo >&2 - echo >&2 "Your PATH variable should have \"/usr/local/opt/gnu-getopt/bin\" in front" + echo >&2 "Login and logout afterwards !!" echo >&2 + echo >&2 "After re-login, your PATH variable should start with \"/usr/local/opt/gnu-getopt/bin\"" echo >&2 "Your current path is ${PATH}" echo >&2 - echo >&2 "Login and logout afterwards !!" - echo >&2 else echo >&2 "You do not have necessary tools in your path (getopt, stat, md5sum)." echo >&2 "Please install latest/GNU version of getopt and coreutils." echo >&2 "This can usually be done with 'apt install util-linux coreutils'" fi - echo + print_info exit 1 fi } @@ -296,211 +316,157 @@ function force_python_3_5() { export PYTHON_VERSION } -# -# Rebuilds the slim image for static checks if needed. In order to speed it up, it's built without NPM -# -function rebuild_image_if_needed_for_static_checks() { - export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="false" - export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" - export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" - export AIRFLOW_CONTAINER_PUSH_IMAGES="false" - export AIRFLOW_CONTAINER_BUILD_NPM="false" # Skip NPM builds to make them faster ! +function confirm_image_rebuild() { + set +e + "${MY_DIR}/../../confirm" "The image ${THE_IMAGE_TYPE} might need to be rebuild." + RES=$? + set -e + if [[ ${RES} == "1" ]]; then + SKIP_REBUILD="true" + elif [[ ${RES} == "2" ]]; then + echo >&2 + echo >&2 "#############################################" + echo >&2 " ERROR! The image require rebuilding. " + echo >&2 "#############################################" + echo >&2 + echo >&2 " You should re-run your command with REBUILD=true environment variable set" + echo >&2 + echo >&2 " * 'REBUILD=true git commit'" + echo >&2 " * 'REBUILD=true git push'" + echo >&2 + echo >&2 " In case you do not want to rebuild, You can always commit the code " + echo >&2 " with --no-verify switch. This skips pre-commit checks. CI will run the tests anyway." + echo >&2 + echo >&2 " You can also rebuild the image: './scripts/ci/local_ci_build.sh'" + echo >&2 " Or pull&build the image from registry: './scripts/ci/local_ci_pull_and_build.sh'" + echo >&2 + exit 1 + else + # Assume Yes also for subsequent questions + export ASSUME_YES_TO_ALL_QUESTIONS="true" + fi +} - export PYTHON_VERSION=3.5 # Always use python version 3.5 for static checks +function rebuild_image_if_needed() { + PYTHON_VERSION=${PYTHON_VERSION:=$(python -c \ + 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')} + export PYTHON_VERSION AIRFLOW_VERSION=$(cat airflow/version.py - << EOF | python print(version.replace("+","")) EOF ) export AIRFLOW_VERSION - export THE_IMAGE="SLIM_CI" - if [[ -f "${BUILD_CACHE_DIR}/.built_${THE_IMAGE}_${PYTHON_VERSION}" ]]; then - if [[ ${AIRFLOW_CONTAINER_FORCE_PULL_IMAGES:=""} != "true" ]]; then - echo - echo "Image built locally - skip force-pulling them" - echo - fi + if [[ -f "${BUILD_CACHE_DIR}/.built_${THE_IMAGE_TYPE}_${PYTHON_VERSION}" ]]; then + print_info + print_info "Image ${THE_IMAGE_TYPE} built locally - skip force-pulling them" + print_info else - echo - echo "Image not built locally - force pulling them first" - echo + print_info + print_info "Image ${THE_IMAGE_TYPE} not built locally - force pulling them first" + print_info export AIRFLOW_CONTAINER_FORCE_PULL_IMAGES="true" export AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED="true" fi AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED=${AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED:="false"} check_if_docker_build_is_needed - if [[ "${AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED}" == "true" ]]; then - local SKIP_REBUILD="false" + SKIP_REBUILD="false" if [[ ${CI:=} != "true" ]]; then - set +e - if ! "${MY_DIR}/../../confirm" "The image might need to be rebuild."; then - SKIP_REBUILD="true" - fi - set -e + confirm_image_rebuild fi if [[ ${SKIP_REBUILD} != "true" ]]; then - echo - echo "Rebuilding image" - echo + print_info + print_info "Rebuilding image" + print_info # shellcheck source=../../hooks/build ./hooks/build | tee -a "${OUTPUT_LOG}" update_all_md5_files - echo - echo "Image rebuilt" - echo + print_info + print_info "Image rebuilt" + print_info fi else - echo - echo "No need to rebuild the image as none of the sensitive files changed: ${FILES_FOR_REBUILD_CHECK}" - echo + print_info + print_info "No need to rebuild the image as none of the sensitive files changed: ${FILES_FOR_REBUILD_CHECK}" + print_info fi +} + +# +# Rebuilds the slim image for static checks if needed. In order to speed it up, it's built without NPM +# +function rebuild_image_if_needed_for_static_checks() { + export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="false" + export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" + export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" + export AIRFLOW_CONTAINER_BUILD_NPM="false" # Skip NPM builds to make them faster ! + + export PYTHON_VERSION=3.5 # Always use python version 3.5 for static checks + + export THE_IMAGE_TYPE="SLIM_CI" + + rebuild_image_if_needed AIRFLOW_SLIM_CI_IMAGE=$(cat "${BUILD_CACHE_DIR}/.AIRFLOW_SLIM_CI_IMAGE") export AIRFLOW_SLIM_CI_IMAGE } +# +# Rebuilds the image for static checks if needed. +# function rebuild_image_if_needed_for_tests() { export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false" - PYTHON_VERSION=${PYTHON_VERSION:=$(python -c \ - 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')} - export PYTHON_VERSION - AIRFLOW_VERSION=$(cat airflow/version.py - << EOF | python -print(version.replace("+","")) -EOF - ) - export AIRFLOW_VERSION - - export THE_IMAGE="CI" - if [[ -f "${BUILD_CACHE_DIR}/.built_${THE_IMAGE}_${PYTHON_VERSION}" ]]; then - echo - echo "Image built locally - skip force-pulling them" - echo - else - echo - echo "Image not built locally - force pulling them first" - echo - export AIRFLOW_CONTAINER_FORCE_PULL_IMAGES="true" - export AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED="true" - fi - export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} - export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} - export AIRFLOW_CONTAINER_PUSH_IMAGES="false" - export AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="true" + export THE_IMAGE_TYPE="CI" - AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED=${AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED:="false"} - check_if_docker_build_is_needed - - if [[ "${AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED}" == "true" ]]; then - local SKIP_REBUILD="false" - if [[ ${CI:=} != "true" ]]; then - set +e - if ! "${MY_DIR}/../../confirm" "The image might need to be rebuild."; then - SKIP_REBUILD="true" - fi - set -e - fi - if [[ ${SKIP_REBUILD} != "true" ]]; then - echo - echo "Rebuilding image" - echo - # shellcheck source=../../hooks/build - ./hooks/build | tee -a "${OUTPUT_LOG}" - update_all_md5_files - echo - echo "Image rebuilt" - echo - fi - else - echo - echo "No need to rebuild the image as none of the sensitive files changed: ${FILES_FOR_REBUILD_CHECK}" - echo - fi + rebuild_image_if_needed AIRFLOW_CI_IMAGE=$(cat "${BUILD_CACHE_DIR}/.AIRFLOW_CI_IMAGE") export AIRFLOW_CI_IMAGE } +# +# Rebuilds the image for licence checks if needed. +# function rebuild_image_if_needed_for_checklicence() { export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true" export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="false" export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true" - export AIRFLOW_CONTAINER_PUSH_IMAGES="false" - PYTHON_VERSION=${PYTHON_VERSION:=$(python -c \ - 'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')} - export PYTHON_VERSION - export THE_IMAGE="CHECKLICENCE" - if [[ -f "${BUILD_CACHE_DIR}/.built_${THE_IMAGE}_${PYTHON_VERSION}" ]]; then - echo - echo "Image built locally - skip force-pulling them" - echo - else - echo - echo "Image not built locally - force pulling them first" - echo - export AIRFLOW_CONTAINER_FORCE_PULL_IMAGES="true" - export AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED="true" - fi - - AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED=${AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED:="false"} - check_if_docker_build_is_needed + export THE_IMAGE_TYPE="CHECKLICENCE" - if [[ "${AIRFLOW_CONTAINER_DOCKER_BUILD_NEEDED}" == "true" ]]; then - local SKIP_REBUILD="false" - if [[ ${CI:=} != "true" ]]; then - set +e - if ! "${MY_DIR}/../../confirm" "The image might need to be rebuild."; then - SKIP_REBUILD="true" - fi - set -e - fi - if [[ ${SKIP_REBUILD} != "true" ]]; then - echo - echo "Rebuilding image" - echo - # shellcheck source=../../hooks/build - ./hooks/build | tee -a "${OUTPUT_LOG}" - update_all_md5_files - echo - echo "Image rebuilt" - echo - fi - else - echo - echo "No need to rebuild the image as none of the sensitive files changed: ${FILES_FOR_REBUILD_CHECK}" - echo - fi + rebuild_image_if_needed AIRFLOW_CHECKLICENCE_IMAGE=$(cat "${BUILD_CACHE_DIR}/.AIRFLOW_CHECKLICENCE_IMAGE") export AIRFLOW_CHECKLICENCE_IMAGE } + # # Starts the script/ If VERBOSE variable is set to true, it enables verbose output of commands executed # Also prints some useful diagnostics information at start of the script # function script_start { - echo - echo "Running $(basename $0)" - echo - echo "Log is redirected to ${OUTPUT_LOG}" - echo + print_info + print_info "Running $(basename $0)" + print_info + print_info "Log is redirected to ${OUTPUT_LOG}" + print_info if [[ ${VERBOSE:=} == "true" ]]; then - echo - echo "Variable VERBOSE Set to \"true\"" - echo "You will see a lot of output" - echo + print_info + print_info "Variable VERBOSE Set to \"true\"" + print_info "You will see a lot of output" + print_info set -x else - echo "You can increase verbosity by running 'export VERBOSE=\"true\"" + print_info "You can increase verbosity by running 'export VERBOSE=\"true\"" if [[ ${SKIP_CACHE_DELETION:=} != "true" ]]; then - echo "And skip deleting the output file with 'export SKIP_CACHE_DELETION=\"true\"" + print_info "And skip deleting the output file with 'export SKIP_CACHE_DELETION=\"true\"" fi - echo + print_info fi START_SCRIPT_TIME=$(date +%s) } @@ -514,18 +480,18 @@ function script_end { fi END_SCRIPT_TIME=$(date +%s) RUN_SCRIPT_TIME=$((END_SCRIPT_TIME-START_SCRIPT_TIME)) - echo - echo "Finished the script $(basename $0)" - echo "It took ${RUN_SCRIPT_TIME} seconds" - echo + print_info + print_info "Finished the script $(basename $0)" + print_info "It took ${RUN_SCRIPT_TIME} seconds" + print_info } function go_to_airflow_sources { - echo + print_info pushd "${MY_DIR}/../../" &>/dev/null || exit 1 - echo - echo "Running in host in $(pwd)" - echo + print_info + print_info "Running in host in $(pwd)" + print_info } # @@ -537,3 +503,103 @@ function basic_sanity_checks() { check_if_coreutils_installed create_cache_directory } + + +function run_flake8() { + FILES=("$@") + + if [[ "${#FILES[@]}" == "0" ]]; then + docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint /opt/airflow/scripts/ci/in_container/run_flake8.sh \ + --env PYTHONDONTWRITEBYTECODE \ + --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ + --env AIRFLOW_CI_SILENT \ + --env HOST_USER_ID="$(id -ur)" \ + --env HOST_GROUP_ID="$(id -gr)" \ + "${AIRFLOW_SLIM_CI_IMAGE}" | tee -a "${OUTPUT_LOG}" + else + docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint /opt/airflow/scripts/ci/in_container/run_flake8.sh \ + --env PYTHONDONTWRITEBYTECODE \ + --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ + --env AIRFLOW_CI_SILENT \ + --env HOST_USER_ID="$(id -ur)" \ + --env HOST_GROUP_ID="$(id -gr)" \ + "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${FILES[@]}" | tee -a "${OUTPUT_LOG}" + fi +} + +function run_docs() { + docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ + --entrypoint /opt/airflow/docs/build.sh \ + --env PYTHONDONTWRITEBYTECODE \ + --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ + --env AIRFLOW_CI_SILENT \ + --env HOST_USER_ID="$(id -ur)" \ + --env HOST_GROUP_ID="$(id -gr)" \ + "${AIRFLOW_SLIM_CI_IMAGE}" +} + +function run_check_license() { + docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ + --entrypoint /opt/airflow/scripts/ci/in_container/run_check_licence.sh \ + --env PYTHONDONTWRITEBYTECODE \ + --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ + --env AIRFLOW_CI_SILENT \ + --env HOST_USER_ID="$(id -ur)" \ + --env HOST_GROUP_ID="$(id -gr)" \ + "${AIRFLOW_CI_IMAGE}" +} + +function run_mypy() { + FILES=("$@") + if [[ "${#FILES[@]}" == "0" ]]; then + docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint /opt/airflow/scripts/ci/in_container/run_mypy.sh \ + --env PYTHONDONTWRITEBYTECODE \ + --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ + --env AIRFLOW_CI_SILENT \ + --env HOST_USER_ID="$(id -ur)" \ + --env HOST_GROUP_ID="$(id -gr)" \ + "${AIRFLOW_SLIM_CI_IMAGE}" \ + "airflow" "tests" "docs" | tee -a "${OUTPUT_LOG}" + else + docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint /opt/airflow/scripts/ci/in_container/run_mypy.sh \ + --env PYTHONDONTWRITEBYTECODE \ + --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ + --env AIRFLOW_CI_SILENT \ + --env HOST_USER_ID="$(id -ur)" \ + --env HOST_GROUP_ID="$(id -gr)" \ + "${AIRFLOW_SLIM_CI_IMAGE}" \ + "${FILES[@]}" | tee -a "${OUTPUT_LOG}" + fi +} + +function run_docker_lint() { + FILES=("$@") + if [[ "${#FILES[@]}" == "0" ]]; then + echo + echo "Running docker lint for all Dockerfiles" + echo + docker run \ + -v "$(pwd):/root" \ + -w /root \ + hadolint/hadolint /bin/hadolint Dockerfile* + echo + echo "Docker pylint completed with no errors" + echo + else + echo + echo "Running docker lint for $*" + echo + docker run \ + -v "$(pwd):/root" \ + -w /root \ + hadolint/hadolint /bin/hadolint "$@" + echo + echo "Docker pylint completed with no errors" + echo + fi +} diff --git a/scripts/ci/ci_before_install.sh b/scripts/ci/ci_before_install.sh index c5bc51c870b237..22144941ecbe75 100755 --- a/scripts/ci/ci_before_install.sh +++ b/scripts/ci/ci_before_install.sh @@ -36,7 +36,7 @@ docker system prune --all --force if [[ ${TRAVIS_JOB_NAME} == "Tests"* ]]; then rebuild_image_if_needed_for_tests -elif [[ ${TRAVIS_JOB_NAME} == "Check license header" ]]; then +elif [[ ${TRAVIS_JOB_NAME} == "Check license headers" ]]; then rebuild_image_if_needed_for_checklicence else rebuild_image_if_needed_for_static_checks @@ -50,4 +50,6 @@ if [[ "${KUBERNETES_VERSION}" == "" ]]; then sudo service docker restart fi +pip install pre-commit + script_end diff --git a/scripts/ci/ci_check_license.sh b/scripts/ci/ci_check_license.sh index 852130085a753a..cbb2ab9327c99c 100755 --- a/scripts/ci/ci_check_license.sh +++ b/scripts/ci/ci_check_license.sh @@ -21,6 +21,8 @@ set -uo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export PYTHON_VERSION=3.5 +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} +export ASSUME_QUIT_TO_ALL_QUESTIONS=${ASSUME_QUIT_TO_ALL_QUESTIONS:="true"} # shellcheck source=./_utils.sh . "${MY_DIR}/_utils.sh" @@ -35,6 +37,7 @@ rebuild_image_if_needed_for_checklicence docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ + --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ "${AIRFLOW_CHECKLICENCE_IMAGE}" diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/ci_docs.sh index 050c6db996dad8..2b5aecf2f15ef5 100755 --- a/scripts/ci/ci_docs.sh +++ b/scripts/ci/ci_docs.sh @@ -20,6 +20,9 @@ set -euo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} +export ASSUME_QUIT_TO_ALL_QUESTIONS=${ASSUME_QUIT_TO_ALL_QUESTIONS:="true"} + # shellcheck source=./_utils.sh . "${MY_DIR}/_utils.sh" @@ -31,12 +34,6 @@ script_start rebuild_image_if_needed_for_static_checks -docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ - --entrypoint /opt/airflow/docs/build.sh \ - --env PYTHONDONTWRITEBYTECODE="true" \ - --env AIRFLOW_CI_VERBOSE=${VERBOSE} \ - --env HOST_USER_ID="$(id -ur)" \ - --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ +run_docs script_end diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/ci_flake8.sh index f79e465ca0d23b..c24ec57d0f1178 100755 --- a/scripts/ci/ci_flake8.sh +++ b/scripts/ci/ci_flake8.sh @@ -20,6 +20,9 @@ set -euo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} +export ASSUME_QUIT_TO_ALL_QUESTIONS=${ASSUME_QUIT_TO_ALL_QUESTIONS:="true"} + # shellcheck source=./_utils.sh . "${MY_DIR}/_utils.sh" @@ -31,25 +34,6 @@ script_start rebuild_image_if_needed_for_static_checks -FILES=("$@") - -if [[ "${#FILES[@]}" == "0" ]]; then - docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint /opt/airflow/scripts/ci/in_container/run_flake8.sh \ - --env PYTHONDONTWRITEBYTECODE="true" \ - --env AIRFLOW_CI_VERBOSE=${VERBOSE} \ - --env HOST_USER_ID="$(id -ur)" \ - --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_SLIM_CI_IMAGE}" | tee -a "${OUTPUT_LOG}" -else - docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint /opt/airflow/scripts/ci/in_container/run_flake8.sh \ - --env PYTHONDONTWRITEBYTECODE="true" \ - --env AIRFLOW_CI_VERBOSE=${VERBOSE} \ - --env HOST_USER_ID="$(id -ur)" \ - --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ - "${FILES[@]}" | tee -a "${OUTPUT_LOG}" -fi +run_flake8 "$@" script_end diff --git a/scripts/ci/ci_lint_dockerfile.sh b/scripts/ci/ci_lint_dockerfile.sh index 29472ca9e41803..ae7a3baeedd28f 100755 --- a/scripts/ci/ci_lint_dockerfile.sh +++ b/scripts/ci/ci_lint_dockerfile.sh @@ -18,6 +18,8 @@ set -euo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} + # shellcheck source=./_utils.sh . "${MY_DIR}/_utils.sh" @@ -25,7 +27,6 @@ basic_sanity_checks script_start -docker run -v "$(pwd)/Dockerfile:/root/Dockerfile" -v "$(pwd)/.hadolint.yaml:/root/.hadolint.yaml" \ - -w /root hadolint/hadolint /bin/hadolint Dockerfile +run_docker_lint "$@" script_end diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/ci_mypy.sh index e44fbd7fbbf5aa..7cc63b861c5d6b 100755 --- a/scripts/ci/ci_mypy.sh +++ b/scripts/ci/ci_mypy.sh @@ -20,6 +20,9 @@ set -euo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} +export ASSUME_QUIT_TO_ALL_QUESTIONS=${ASSUME_QUIT_TO_ALL_QUESTIONS:="true"} + # shellcheck source=./_utils.sh . "${MY_DIR}/_utils.sh" @@ -31,25 +34,6 @@ script_start rebuild_image_if_needed_for_static_checks -FILES=("$@") -if [[ "${#FILES[@]}" == "0" ]]; then - docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint /opt/airflow/scripts/ci/in_container/run_mypy.sh \ - --env PYTHONDONTWRITEBYTECODE="true" \ - --env AIRFLOW_CI_VERBOSE=${VERBOSE} \ - --env HOST_USER_ID="$(id -ur)" \ - --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ - "airflow" "tests" | tee -a "${OUTPUT_LOG}" -else - docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint /opt/airflow/scripts/ci/in_container/run_mypy.sh \ - --env PYTHONDONTWRITEBYTECODE="true" \ - --env AIRFLOW_CI_VERBOSE=${VERBOSE} \ - --env HOST_USER_ID="$(id -ur)" \ - --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_SLIM_CI_IMAGE}" \ - "${FILES[@]}" | tee -a "${OUTPUT_LOG}" -fi +run_mypy "$@" script_end diff --git a/scripts/ci/ci_build.sh b/scripts/ci/ci_pylint_main.sh old mode 100644 new mode 100755 similarity index 85% rename from scripts/ci/ci_build.sh rename to scripts/ci/ci_pylint_main.sh index ac4abe322d31ff..c30d4f65386638 --- a/scripts/ci/ci_build.sh +++ b/scripts/ci/ci_pylint_main.sh @@ -20,13 +20,20 @@ set -euo pipefail MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} +export ASSUME_QUIT_TO_ALL_QUESTIONS=${ASSUME_QUIT_TO_ALL_QUESTIONS:="true"} + # shellcheck source=./_utils.sh . "${MY_DIR}/_utils.sh" basic_sanity_checks +force_python_3_5 + script_start rebuild_image_if_needed_for_static_checks +run_pylint_main "$@" + script_end diff --git a/scripts/ci/ci_pylint_tests.sh b/scripts/ci/ci_pylint_tests.sh new file mode 100755 index 00000000000000..2fd2992a45bc21 --- /dev/null +++ b/scripts/ci/ci_pylint_tests.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} +export ASSUME_QUIT_TO_ALL_QUESTIONS=${ASSUME_QUIT_TO_ALL_QUESTIONS:="true"} + +# shellcheck source=./_utils.sh +. "${MY_DIR}/_utils.sh" + +basic_sanity_checks + +force_python_3_5 + +script_start + +rebuild_image_if_needed_for_static_checks + +run_pylint_tests "$@" + +script_end diff --git a/scripts/ci/ci_run_airflow_testing.sh b/scripts/ci/ci_run_airflow_testing.sh index bcdb7ec9ed5e62..0b22e1e798bece 100755 --- a/scripts/ci/ci_run_airflow_testing.sh +++ b/scripts/ci/ci_run_airflow_testing.sh @@ -32,13 +32,19 @@ script_start rebuild_image_if_needed_for_tests +# Test environment export BACKEND=${BACKEND:="sqlite"} export ENV=${ENV:="docker"} export KUBERNETES_MODE=${KUBERNETES_MODE:="git_mode"} + +# Whether local sources are mounted to docker export MOUNT_LOCAL_SOURCES=${MOUNT_LOCAL_SOURCES:="false"} -export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="8080"} + +# whethere verbose output should be produced export AIRFLOW_CI_VERBOSE=${VERBOSE} -export PYTHONDONTWRITEBYTECODE="true" + +# opposite - whether diagnostict messages should be silenced +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} if [[ ${MOUNT_LOCAL_SOURCES} == "true" ]]; then DOCKER_COMPOSE_LOCAL=("-f" "${MY_DIR}/docker-compose-local.yml") diff --git a/scripts/ci/ci_run_all_static_tests.sh b/scripts/ci/ci_run_all_static_tests.sh new file mode 100755 index 00000000000000..9fb356af45d17a --- /dev/null +++ b/scripts/ci/ci_run_all_static_tests.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export AIRFLOW_CI_SILENT=${AIRFLOW_CI_SILENT:="true"} +export ASSUME_QUIT_TO_ALL_QUESTIONS=${ASSUME_QUIT_TO_ALL_QUESTIONS:="true"} + +# shellcheck source=./_utils.sh +. "${MY_DIR}/_utils.sh" + +basic_sanity_checks + +force_python_3_5 + +script_start + +rebuild_image_if_needed_for_static_checks + +SKIP=pylint pre-commit run --all-files + +script_end diff --git a/scripts/ci/docker-compose-kubernetes.yml b/scripts/ci/docker-compose-kubernetes.yml index 41b411f91b8b93..cb157043bb68d1 100644 --- a/scripts/ci/docker-compose-kubernetes.yml +++ b/scripts/ci/docker-compose-kubernetes.yml @@ -1,19 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- version: "2.2" services: airflow-testing: diff --git a/scripts/ci/docker-compose-local.yml b/scripts/ci/docker-compose-local.yml index 4a4741e85d1163..95c47ee5991374 100644 --- a/scripts/ci/docker-compose-local.yml +++ b/scripts/ci/docker-compose-local.yml @@ -1,22 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- version: "2.2" services: airflow-testing: + # We need to mount files an directories individually because some files + # such apache_airflow.egg-info should not be mounted from host + # we only mount those files that it makes sense to edit while developing + # or those that might be useful to see in the host as output of the + # tests (such as logs) volumes: - ../../airflow:/opt/airflow/airflow:cached - ../../setup.cfg:/opt/airflow/setup.cfg:cached @@ -36,9 +42,15 @@ services: - ../../.flake8:/opt/airflow/.flake8:cached - ../../.rat-excludes:/opt/airflow/.rat-excludes:cached - ../../run-tests:/opt/airflow/run-tests:cached + - ../../run-tests-complete:/root/run-tests-complete:cached - ../../logs:/root/airflow/logs:cached - ../../logs:/opt/airflow/logs:cached - ../../tmp:/opt/airflow/tmp:cached + - ../../.bash_completion.d:/root/.bash_completion.d:cached + - ../../.bash_completion:/root/.bash_completion:cached + - ../../.bash_history:/root/.bash_history:cached + - ../../.bash_aliases:/root/.bash_aliases:cached + - ../../.inputrc:/root/.inputrc:cached environment: - HOST_USER_ID - HOST_GROUP_ID diff --git a/scripts/ci/docker-compose.yml b/scripts/ci/docker-compose.yml index 5020cc13e359a4..57394b8e35e7d8 100644 --- a/scripts/ci/docker-compose.yml +++ b/scripts/ci/docker-compose.yml @@ -89,6 +89,7 @@ services: - TRAVIS_TAG - RUN_TESTS - AIRFLOW_CI_VERBOSE + - AIRFLOW_CI_SILENT depends_on: - postgres - mysql diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh index 240a8f3d2ee1ba..e03325618f8f4d 100644 --- a/scripts/ci/in_container/_in_container_utils.sh +++ b/scripts/ci/in_container/_in_container_utils.sh @@ -44,46 +44,52 @@ function in_container_script_end() { fi } +function print_in_container_info() { + if [[ ${AIRFLOW_CI_SILENT:="false"} != "true" ]]; then + echo "$@" + fi +} + # # Cleans up PYC files (in case they come in mounted folders) # function in_container_cleanup_pyc() { - echo - echo "Cleaning up .pyc files" - echo + print_in_container_info + print_in_container_info "Cleaning up .pyc files" + print_in_container_info set +o pipefail - sudo find . \ + NUM_FILES=$(sudo find . \ -path "./airflow/www/node_modules" -prune -o \ -path "./airflow/www_rbac/node_modules" -prune -o \ -path "./.eggs" -prune -o \ -path "./docs/_build" -prune -o \ -path "./build" -prune -o \ - -name "*.pyc" | grep ".pyc$" | sudo xargs rm -vf | wc -l | \ - xargs -n 1 echo "Number of deleted .pyc files:" + -name "*.pyc" | grep ".pyc$" | sudo xargs rm -vf | wc -l) + print_in_container_info "Number of deleted .pyc files:" set -o pipefail - echo - echo + print_in_container_info + print_in_container_info } # # Cleans up __pycache__ directories (in case they come in mounted folders) # function in_container_cleanup_pycache() { - echo - echo "Cleaning up __pycache__ directories" - echo + print_in_container_info + print_in_container_info "Cleaning up __pycache__ directories" + print_in_container_info set +o pipefail - sudo find . \ + NUM_FILES=$(find . \ -path "./airflow/www/node_modules" -prune -o \ -path "./airflow/www_rbac/node_modules" -prune -o \ -path "./.eggs" -prune -o \ -path "./docs/_build" -prune -o \ -path "./build" -prune -o \ - -name "__pycache__" | grep "__pycache__" | sudo xargs rm -rvf | wc -l | \ - xargs -n 1 echo "Number of deleted __pycache__ dirs (and files):" + -name "__pycache__" | grep "__pycache__" | sudo xargs rm -rvf | wc -l) + print_in_container_info "Number of deleted __pycache__ dirs (and files):" set -o pipefail - echo - echo + print_in_container_info + print_in_container_info } # @@ -91,22 +97,22 @@ function in_container_cleanup_pycache() { # The host user. # function in_container_fix_ownership() { - echo - echo "Changing ownership of root-owned files to ${HOST_USER_ID}.${HOST_GROUP_ID}" - echo + print_in_container_info + print_in_container_info "Changing ownership of root-owned files to ${HOST_USER_ID}.${HOST_GROUP_ID}" + print_in_container_info set +o pipefail sudo find . -user root | sudo xargs chown -v "${HOST_USER_ID}.${HOST_GROUP_ID}" | wc -l | \ xargs -n 1 echo "Number of files with changed ownership:" set -o pipefail - echo - echo + print_in_container_info + print_in_container_info } function in_container_go_to_airflow_sources() { pushd "${AIRFLOW_SOURCES}" &>/dev/null || exit 1 - echo - echo "Running in $(pwd)" - echo + print_in_container_info + print_in_container_info "Running in $(pwd)" + print_in_container_info } function in_container_basic_sanity_check() { diff --git a/scripts/ci/in_container/run_check_licence.sh b/scripts/ci/in_container/run_check_licence.sh index b71a00c33c0dd5..03c4b515aa0f1f 100755 --- a/scripts/ci/in_container/run_check_licence.sh +++ b/scripts/ci/in_container/run_check_licence.sh @@ -17,9 +17,7 @@ # limitations under the License. # -# Script to run Pylint on all code. Can be started from any working directory -# ./scripts/ci/run_pylint.sh - +# Script to check licences for all code. Can be started from any working directory set -uo pipefail MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) diff --git a/scripts/ci/in_container/run_docs_build.sh b/scripts/ci/in_container/run_docs_build.sh index c7554063b99c38..c885e2ea13f1aa 100755 --- a/scripts/ci/in_container/run_docs_build.sh +++ b/scripts/ci/in_container/run_docs_build.sh @@ -17,9 +17,7 @@ # limitations under the License. # -# Script to run Pylint on all code. Can be started from any working directory -# ./scripts/ci/run_pylint.sh - +# Script to build docs. Can be started from any working directory set -uo pipefail MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) diff --git a/scripts/ci/in_container/run_flake8.sh b/scripts/ci/in_container/run_flake8.sh index d5b4a1cd749d57..f2fdac8ab27a2a 100755 --- a/scripts/ci/in_container/run_flake8.sh +++ b/scripts/ci/in_container/run_flake8.sh @@ -17,9 +17,7 @@ # limitations under the License. # -# Script to run Pylint on all code. Can be started from any working directory -# ./scripts/ci/run_pylint.sh - +# Script to run flake8 on all code. Can be started from any working directory set -uo pipefail MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) diff --git a/scripts/ci/in_container/run_mypy.sh b/scripts/ci/in_container/run_mypy.sh index 03782ad17af005..20154efae410e1 100755 --- a/scripts/ci/in_container/run_mypy.sh +++ b/scripts/ci/in_container/run_mypy.sh @@ -17,9 +17,7 @@ # limitations under the License. # -# Script to run Pylint on all code. Can be started from any working directory -# ./scripts/ci/run_pylint.sh - +# Script to run mypy on all code. Can be started from any working directory set -uo pipefail MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) diff --git a/scripts/ci/in_container/run_pylint.sh b/scripts/ci/in_container/run_pylint_main.sh similarity index 77% rename from scripts/ci/in_container/run_pylint.sh rename to scripts/ci/in_container/run_pylint_main.sh index 201fab1301b7e6..b28688b7fcb806 100755 --- a/scripts/ci/in_container/run_pylint.sh +++ b/scripts/ci/in_container/run_pylint_main.sh @@ -17,9 +17,7 @@ # limitations under the License. # -# Script to run Pylint on all code. Can be started from any working directory -# ./scripts/ci/run_pylint.sh - +# Script to run Pylint on main code. Can be started from any working directory set -uo pipefail MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) @@ -32,10 +30,6 @@ in_container_basic_sanity_check in_container_script_start if [[ ${#@} == "0" ]]; then - echo - echo "Running Pylint with no parameters" - echo - echo echo "Running pylint for all sources except 'tests' folder" echo @@ -55,32 +49,17 @@ if [[ ${#@} == "0" ]]; then -not -name 'webserver_config.py' | \ grep ".*.py$" | \ grep -vFf scripts/ci/pylint_todo.txt | xargs pylint --output-format=colorized - RES_MAIN=$? - - echo - echo "Running pylint for 'tests' folder" - echo - find "./tests" -name "*.py" | \ - grep -vFf scripts/ci/pylint_todo.txt | \ - xargs pylint --disable=" - missing-docstring, - no-self-use, - too-many-public-methods, - protected-access - " \ - --output-format=colorized - RES_TESTS=$? + RES=$? else echo "Running Pylint with parameters: $*" echo pylint --output-format=colorized "$@" - RES_MAIN=$? - RES_TESTS="0" + RES=$? fi in_container_script_end -if [[ "${RES_TESTS}" != 0 || "${RES_MAIN}" != 0 ]]; then +if [[ "${RES}" != 0 ]]; then echo >&2 echo >&2 "There were some pylint errors. Exiting" echo >&2 diff --git a/scripts/ci/in_container/run_pylint_tests.sh b/scripts/ci/in_container/run_pylint_tests.sh new file mode 100755 index 00000000000000..31e26b1b0aacfd --- /dev/null +++ b/scripts/ci/in_container/run_pylint_tests.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Script to run Pylint on test code. Can be started from any working directory +set -uo pipefail + +MY_DIR=$(cd "$(dirname "$0")" || exit 1; pwd) + +# shellcheck source=./_in_container_utils.sh +. "${MY_DIR}/_in_container_utils.sh" + +in_container_basic_sanity_check + +in_container_script_start + +DISABLE_CHECKS="missing-docstring,no-self-use,too-many-public-methods,protected-access" + +if [[ ${#@} == "0" ]]; then + echo + echo "Running pylint for 'tests' folder" + echo + find "./tests" -name "*.py" | \ + grep -vFf scripts/ci/pylint_todo.txt | \ + xargs pylint --disable="${DISABLE_CHECKS}" --output-format=colorized + RES=$? +else + echo "Running Pylint for tests with parameters: $*" + echo + pylint --disable="${DISABLE_CHECKS}" --output-format=colorized "$@" + RES=$? +fi + +in_container_script_end + +if [[ "${RES}" != 0 ]]; then + echo >&2 + echo >&2 "There were some pylint errors. Exiting" + echo >&2 + exit 1 +else + echo + echo "Pylint check succeeded" + echo +fi diff --git a/scripts/ci/local_ci_build.sh b/scripts/ci/local_ci_build.sh index 9bbe74ca8584f6..588cd413bdba93 100755 --- a/scripts/ci/local_ci_build.sh +++ b/scripts/ci/local_ci_build.sh @@ -31,8 +31,6 @@ basic_sanity_checks script_start -export ASSUME_YES="true" - rebuild_image_if_needed_for_tests rebuild_image_if_needed_for_static_checks diff --git a/scripts/ci/local_ci_fix_ownership.sh b/scripts/ci/local_ci_fix_ownership.sh index 2a317320e393e0..6b888349c7dc4a 100755 --- a/scripts/ci/local_ci_fix_ownership.sh +++ b/scripts/ci/local_ci_fix_ownership.sh @@ -33,10 +33,6 @@ basic_sanity_checks script_start export PYTHON_VERSION=${PYTHON_VERSION:="3.6"} -export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} -export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} -export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="8080"} -export PYTHONDONTWRITEBYTECODE="true" export AIRFLOW_CONTAINER_DOCKER_IMAGE=\ ${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CONTAINER_BRANCH_NAME}-python${PYTHON_VERSION}-ci diff --git a/scripts/ci/local_ci_pull_and_build.sh b/scripts/ci/local_ci_pull_and_build.sh index 3e37a8049d7246..dd92abad995299 100755 --- a/scripts/ci/local_ci_pull_and_build.sh +++ b/scripts/ci/local_ci_pull_and_build.sh @@ -33,7 +33,7 @@ script_start export AIRFLOW_CONTAINER_FORCE_PULL_IMAGES="true" export AIRFLOW_CONTAINER_SKIP_LATEST_PYTHON_PULL="false" -export ASSUME_YES="true" +export ASSUME_YES_TO_ALL_QUESTIONS="true" rebuild_image_if_needed_for_tests diff --git a/scripts/ci/local_ci_stop_environment.sh b/scripts/ci/local_ci_stop_environment.sh index 870f0ae1dd3b60..2969894e673953 100755 --- a/scripts/ci/local_ci_stop_environment.sh +++ b/scripts/ci/local_ci_stop_environment.sh @@ -32,9 +32,6 @@ basic_sanity_checks script_start export PYTHON_VERSION=${PYTHON_VERSION:="3.6"} -export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} -export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} -export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="8080"} # Default branch name for triggered builds is master export AIRFLOW_CONTAINER_BRANCH_NAME=${AIRFLOW_CONTAINER_BRANCH_NAME:="master"} diff --git a/setup.py b/setup.py index a4151906ebdf02..faa7adc5a28946 100644 --- a/setup.py +++ b/setup.py @@ -32,8 +32,7 @@ logger = logging.getLogger(__name__) # noinspection PyUnresolvedReferences -version = imp.load_source( - 'airflow.version', os.path.join('airflow', 'version.py')).version +version = imp.load_source('airflow.version', os.path.join('airflow', 'version.py')).version # type: ignore PY3 = sys.version_info[0] == 3 @@ -63,7 +62,7 @@ class CleanCommand(Command): """ description = "Tidy up the project root" - user_options = [] + user_options = [] # type: ignore def initialize_options(self): """Set default values for options.""" @@ -84,7 +83,7 @@ class CompileAssets(Command): """ description = "Compile and build the frontend assets" - user_options = [] + user_options = [] # type: ignore def initialize_options(self): """Set default values for options.""" @@ -276,6 +275,7 @@ def write_version(filename=os.path.join(*["airflow", "git_version"])): 'nose-timer', 'parameterized', 'paramiko', + 'pre-commit', 'pysftp', 'pywinrm', 'qds-sdk>=1.9.6', diff --git a/tests/dags/test_example_bash_operator.py b/tests/dags/test_example_bash_operator.py index a87db8dd7ccd7d..96a53aa921a527 100644 --- a/tests/dags/test_example_bash_operator.py +++ b/tests/dags/test_example_bash_operator.py @@ -44,9 +44,8 @@ run_this.set_downstream(run_this_last) for i in range(3): - i = str(i) task = BashOperator( - task_id='runme_' + i, + task_id='runme_' + str(i), bash_command='echo "{{ task_instance_key_str }}" && sleep 1', dag=dag) task.set_downstream(run_this)