[AIRFLOW-5161] Static checks are run automatically in pre-commit hooks (

#5777)
apache · Aug 15, 2019 · 58fa6b5 · 58fa6b5
1 parent f483683
commit 58fa6b5
Show file tree

Hide file tree

Showing 35 changed files with 828 additions and 508 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+default_stages: [commit, push]
+repos:
+  - repo: meta
+    hooks:
+      - id: check-hooks-apply
+  - repo: local
+    hooks:
+      - id: lint-dockerfile
+        name: Lint dockerfile
+        language: system
+        entry: ./scripts/ci/ci_lint_dockerfile.sh
+        files: ^Dockerfile.*$
+      - id: mypy
+        name: Run mypy
+        language: system
+        entry: ./scripts/ci/ci_mypy.sh
+        files: \.py$
+        require_serial: true
+      - id: flake8
+        name: Run flake8
+        language: system
+        entry: ./scripts/ci/ci_flake8.sh
+        files: \.py$
+        require_serial: true
diff --git a/.travis.yml b/.travis.yml
@@ -21,6 +21,8 @@ language: python
 env:
   global:
     - BUILD_ID=${TRAVIS_BUILD_ID}
+    - AIRFLOW_CONTAINER_MOUNT_HOST_VOLUMES="false"
+    - AIRFLOW_CI_SILENT="false"
     - CI="true"
 python: "3.6"
 stages:
@@ -36,9 +38,9 @@ jobs:
       env: BACKEND=sqlite ENV=docker
       python: "3.5"
       stage: test
-    - name: "Tests mysql python 2.7"
+    - name: "Tests mysql python 3.6"
       env: BACKEND=mysql ENV=docker
-      python: "2.7"
+      python: "3.6"
       stage: test
     - name: "Tests postgres kubernetes python 3.6 (persistent)"
       env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.13.0 KUBERNETES_MODE=persistent_mode
@@ -56,20 +58,11 @@ jobs:
       env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.9.0 KUBERNETES_MODE=git_mode
       python: "2.7"
       stage: test
-    - name: Flake8
-      stage: pre-test
-      script: ./scripts/ci/ci_flake8.sh
-    - name: mypy
-      stage: pre-test
-      script: ./scripts/ci/ci_mypy.sh
-    - name: Check license header
+    - name: "Static checks"
       stage: pre-test
-      script: ./scripts/ci/ci_check_license.sh
-    - name: Lint Dockerfile
-      stage: pre-test
-      script: ./scripts/ci/ci_lint_dockerfile.sh
+      script: ./scripts/ci/ci_run_all_static_tests.sh
     - name: Check docs
-      stage: test
+      stage: pre-test
       script: ./scripts/ci/ci_docs.sh
 services:
   - docker

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
diff --git a/airflow/contrib/hooks/bigquery_hook.py b/airflow/contrib/hooks/bigquery_hook.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*- # pylint: disable=too-many-lines
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

diff --git a/confirm b/confirm
@@ -17,15 +17,27 @@
 # under the License.
 set -euo pipefail
 
-if [[ "${ASSUME_YES:=false}" == "true" ]]; then
+if [[ "${ASSUME_YES_TO_ALL_QUESTIONS:=false}" == "true" ]]; then
     exit 0
 fi
 
-read -r -p "${1}. Are you sure? [y/N] " response
+if [[ "${ASSUME_NO_TO_ALL_QUESTIONS:=false}" == "true" ]]; then
+    exit 1
+fi
+
+if [[ "${ASSUME_QUIT_TO_ALL_QUESTIONS:=false}" == "true" ]]; then
+    exit 2
+fi
+
+
+read -r -p "${1}. Are you sure? [y/N/q] " response
 case "$response" in
     [yY][eE][sS]|[yY])
         exit 0
         ;;
+    [qQ][uU][iI|[tT]|[qQ])
+        exit 2
+        ;;
     *)
         exit 1
         ;;

diff --git a/docs/conf.py b/docs/conf.py
@@ -31,8 +31,10 @@
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
+"""Configuration of Airflow Docs"""
 import os
 import sys
+from typing import Dict
 
 import airflow
 
@@ -255,7 +257,7 @@
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
-import sphinx_rtd_theme
+import sphinx_rtd_theme  # pylint: disable=wrong-import-position,wrong-import-order
 
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
@@ -340,7 +342,7 @@
 
     # Additional stuff for the LaTeX preamble.
     # 'preamble': '',
-}
+}  # type: Dict[str,str]
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,

diff --git a/hooks/build b/hooks/build
@@ -147,15 +147,19 @@ export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
 # own docker images. In this case you can build images locally and push them
 export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
 
+# You can override python binary with specific version
+export PYTHON_BINARY=${PYTHON_BINARY:=python}
+
 # Determine python version from the local python on the path.
 # You can override it with PYTHON_VERSION because all python version dependencies
 # Are determined in Docker images. If you pass IMAGE_NAME, python version will be extracted from the name
 # This default Python version is later overridden in case IMAGE_NAME is passed
 export PYTHON_VERSION=\
-${PYTHON_VERSION:=$(python -c 'import sys;print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')}
+${PYTHON_VERSION:=$(${PYTHON_BINARY} -c \
+ 'import sys;print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')}
 
 # shellcheck source=./_default_branch.sh
-. ${MY_DIR}/_default_branch.sh
+. "${MY_DIR}/_default_branch.sh"
 
 # Source brnnch will be set in DockerHub
 SOURCE_BRANCH=${SOURCE_BRANCH:=${DEFAULT_BRANCH}}
@@ -259,7 +263,7 @@ AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE=${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_
 
 pwd
 # Determine version of the Airflow from version.py
-AIRFLOW_VERSION=$(cat airflow/version.py - << EOF | python
+AIRFLOW_VERSION=$(cat airflow/version.py - << EOF | ${PYTHON_BINARY}
 print(version.replace("+",""))
 EOF
 )
@@ -268,7 +272,7 @@ export AIRFLOW_VERSION
 # Check if we are running in the CI environment
 CI=${CI:="false"}
 
-if [[ ${CI} == "true" ]]; then
+if [[ "${CI}" == "true" ]]; then
     NON_CI="false"
 else
     NON_CI="true"
@@ -425,57 +429,65 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then
         IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CHECKLICENCE_IMAGE}"
     fi
 
-    for IMAGE in ${IMAGES_TO_PULL}
-    do
+    DOCKER_CACHE_DIRECTIVE_CI=()
+    DOCKER_CACHE_DIRECTIVE_CI_SLIM=()
+    if [[ ${IMAGES_TO_PULL} == "" ]]; then
         echo
-        echo "Checking whether image ${IMAGE} needs to be pulled."
+        echo "Skipping building of all images."
         echo
-        PULL_IMAGE="false"
-        if [[ "${AIRFLOW_CONTAINER_FORCE_PULL_IMAGES}" == "true" ]]; then
+    else
+        for IMAGE in ${IMAGES_TO_PULL}
+        do
             echo
-            echo "Pulling images is forced. Pulling ${IMAGE}"
+            echo "Checking whether image ${IMAGE} needs to be pulled."
             echo
-            PULL_IMAGE="true"
-        else
-            IMAGE_HASH=$(docker images -q "${IMAGE}" 2> /dev/null)
-            if [[ "${IMAGE_HASH}" == "" ]]; then
+            PULL_IMAGE="false"
+            if [[ "${AIRFLOW_CONTAINER_FORCE_PULL_IMAGES}" == "true" ]]; then
                 echo
-                echo "No image ${IMAGE} locally available. Pulling for the first time."
+                echo "Pulling images is forced. Pulling ${IMAGE}"
                 echo
                 PULL_IMAGE="true"
             else
-                echo
-                echo "Image ${IMAGE} is in local registry (${IMAGE_HASH}). Not pulling it!"
-                echo
-                PULL_IMAGE="false"
+                IMAGE_HASH=$(docker images -q "${IMAGE}" 2> /dev/null)
+                if [[ "${IMAGE_HASH}" == "" ]]; then
+                    echo
+                    echo "No image ${IMAGE} locally available. Pulling for the first time."
+                    echo
+                    PULL_IMAGE="true"
+                else
+                    echo
+                    echo "Image ${IMAGE} is in local registry (${IMAGE_HASH}). Not pulling it!"
+                    echo
+                    PULL_IMAGE="false"
+                fi
             fi
-        fi
-        if [[ "${PULL_IMAGE}" == "true" ]]; then
-            if [[ "${AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES}" == "true" ]]; then
-                echo
-                echo "Skipping pulling image ${IMAGE}"
-                echo
+            if [[ "${PULL_IMAGE}" == "true" ]]; then
+                if [[ "${AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES}" == "true" ]]; then
+                    echo
+                    echo "Skipping pulling image ${IMAGE}"
+                    echo
+                else
+                    echo
+                    set -x
+                    docker pull "${IMAGE}" || true
+                    set +x
+                    echo
+                fi
+            fi
+            if [[ "${IMAGE}" == "${AIRFLOW_CI_IMAGE}" ]]; then
+                DOCKER_CACHE_DIRECTIVE_CI+=("--cache-from" "${IMAGE}")
+            elif [[ "${IMAGE}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then
+                DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--cache-from" "${IMAGE}")
+            elif [[ "${IMAGE}" == "${AIRFLOW_CHECKLICENCE_IMAGE}" ]]; then
+                DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--cache-from" "${IMAGE}")
             else
                 echo
-                set -x
-                docker pull "${IMAGE}" || true
-                set +x
+                echo "Don't know how to set cache directive for ${IMAGE}. Exiting"
                 echo
+                exit 1
             fi
-        fi
-        if [[ "${IMAGE}" == "${AIRFLOW_CI_IMAGE}" ]]; then
-            DOCKER_CACHE_DIRECTIVE_CI+=("--cache-from" "${IMAGE}")
-        elif [[ "${IMAGE}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then
-            DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--cache-from" "${IMAGE}")
-        elif [[ "${IMAGE}" == "${AIRFLOW_CHECKLICENCE_IMAGE}" ]]; then
-            DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--cache-from" "${IMAGE}")
-        else
-            echo
-            echo "Don't know how to set cache directive for ${IMAGE}. Exiting"
-            echo
-            exit 1
-        fi
-    done
+        done
+    fi
 fi
 
 start_step "Setting cache options"
@@ -600,13 +612,16 @@ else
     if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then
         docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_DEFAULT}"
         add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_DEFAULT}"
+        save_to_file AIRFLOW_SLIM_CI_IMAGE_DEFAULT
     fi
     if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then
         docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_LATEST}"
         add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_LATEST}"
+        save_to_file AIRFLOW_SLIM_CI_IMAGE_LATEST
         if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then
             docker tag "${AIRFLOW_SLIM_CI_IMAGE}" "${AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT}"
             add_image_to_push "${AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT}"
+            save_to_file AIRFLOW_SLIM_CI_IMAGE_LATEST_DEFAULT
         fi
     fi
 fi
@@ -627,13 +642,16 @@ else
     if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then
         docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_IMAGE_DEFAULT}"
         add_image_to_push "${AIRFLOW_CI_IMAGE_DEFAULT}"
+        save_to_file AIRFLOW_CI_IMAGE_DEFAULT
     fi
     if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then
         docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_IMAGE_LATEST}"
         add_image_to_push "${AIRFLOW_CI_IMAGE_LATEST}"
+        save_to_file AIRFLOW_CI_IMAGE_LATEST
         if [[ "${PYTHON_VERSION_FOR_DEFAULT_IMAGE}" == "${PYTHON_VERSION}" ]]; then
             docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_IMAGE_LATEST_DEFAULT}"
             add_image_to_push "${AIRFLOW_CI_IMAGE_LATEST_DEFAULT}"
+            save_to_file AIRFLOW_CI_IMAGE_LATEST_DEFAULT
         fi
     fi
 fi

diff --git a/hooks/push b/hooks/push
@@ -1,25 +1,20 @@
 #!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing,
-#  software distributed under the License is distributed on an
-#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-#  KIND, either express or implied.  See the License for the
-#  specific language governing permissions and limitations
-#  under the License.
-
-# This is hook build used by DockerHub. We are also using it
-# on Travis CI to potentially rebuild (and refresh layers that
-# are not cached) Docker images that are used to run CI jobs
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 # We do not push in the push step because we are building multiple images in the build step
 # and it is difficult to pass list of the built images from the build to push phase