[AIRFLOW-6066] Added pre-commit checks for accidental debug stmts (ap…

…ache#6662)

(cherry picked from commit 0ff9e23)

.pre-commit-config.yaml

@@ -143,6 +143,7 @@ repos:
     rev: v2.4.0
     hooks:
       - id: check-merge-conflict
+      - id: debug-statements
       - id: detect-private-key
       - id: end-of-file-fixer
       - id: mixed-line-ending
@@ -227,6 +228,12 @@ repos:
         files: \.py$
         pass_filenames: true
         require_serial: true
+      - id: pydevd
+        language: pygrep
+        name: Check for pydevd debug statements accidentally left
+        entry: "pydevd.*settrace\\("
+        pass_filenames: true
+        files: \.py$
       - id: build
         name: Check if image build is needed
         entry: ./scripts/ci/pre_commit_ci_build.sh

.travis.yml

@@ -16,7 +16,7 @@
 # under the License.
 #
 ---
-dist: xenial
+dist: bionic
 language: python
 env:
   global:
@@ -33,34 +33,29 @@ stages:
 jobs:
   include:
     - name: "Static checks"
-      python: "3.5"
       stage: pre-test
       script: ./scripts/ci/ci_run_all_static_tests.sh
       env: AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="true"
     - name: "Build documentation"
-      python: "3.5"
       stage: pre-test
       script: ./scripts/ci/ci_docs.sh
     - name: "Tests postgres python 3.6"
       env: >-
         BACKEND=postgres
         ENV=docker
         PYTHON_VERSION=3.6
-      python: "3.6"
       stage: test
     - name: "Tests sqlite python 3.5"
       env: >-
         BACKEND=sqlite
         ENV=docker
         PYTHON_VERSION=3.5
-      python: "3.5"
       stage: test
     - name: "Tests mysql python 2.7"
       env: >-
         BACKEND=mysql
         ENV=docker
         PYTHON_VERSION=2.7
-      python: "2.7"
       stage: test
     - name: "Tests postgres kubernetes python 3.6 (persistent)"
       env: >-
@@ -69,7 +64,6 @@ jobs:
         KUBERNETES_VERSION=v1.15.3
         KUBERNETES_MODE=persistent_mode
         PYTHON_VERSION=3.6
-      python: "3.6"
       stage: test
       script: travis_wait 30 "./scripts/ci/ci_run_airflow_testing.sh"
     - name: "Tests postgres kubernetes python 3.6 (git)"
@@ -79,7 +73,6 @@ jobs:
         KUBERNETES_VERSION=v1.15.3
         KUBERNETES_MODE=git_mode
         PYTHON_VERSION=3.6
-      python: "3.6"
       stage: test
       script: travis_wait 30 "./scripts/ci/ci_run_airflow_testing.sh"
     - name: "Tests postgres kubernetes python 2.7 (persistent)"
@@ -89,7 +82,6 @@ jobs:
         KUBERNETES_VERSION=v1.15.3
         KUBERNETES_MODE=persistent_mode
         PYTHON_VERSION=2.7
-      python: "2.7"
       stage: test
       script: travis_wait 30 "./scripts/ci/ci_run_airflow_testing.sh"
     - name: "Tests postgres kubernetes python 2.7 (git)"
@@ -99,7 +91,6 @@ jobs:
         KUBERNETES_VERSION=v1.15.3
         KUBERNETES_MODE=git_mode
         PYTHON_VERSION=2.7
-      python: "2.7"
       stage: test
       script: travis_wait 30 "./scripts/ci/ci_run_airflow_testing.sh"
   services:

BREEZE.rst

@@ -872,7 +872,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
   -S, --static-check <STATIC_CHECK>
           Run selected static checks for currently changed files. You should specify static check that
           you would like to run or 'all' to run all checks. One of
-          [ all check-hooks-apply check-merge-conflict check-executables-have-shebangs check-xml detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license check-apache-license lint-dockerfile mixed-line-ending mypy shellcheck].
+          [ all check-hooks-apply check-merge-conflict check-executables-have-shebangs check-xml debug-statements detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license check-apache-license lint-dockerfile mixed-line-ending mypy shellcheck].
           You can pass extra arguments including options to to the pre-commit framework as
           <EXTRA_ARGS> passed after --. For example:
 
@@ -886,7 +886,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
   -F, --static-check-all-files <STATIC_CHECK>
           Run selected static checks for all applicable files. You should specify static check that
           you would like to run or 'all' to run all checks. One of
-          [ all check-hooks-apply check-merge-conflict check-executables-have-shebangs check-xml detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license check-apache-license lint-dockerfile mixed-line-ending mypy shellcheck].
+          [ all check-hooks-apply check-merge-conflict check-executables-have-shebangs check-xml debug-statements detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license check-apache-license lint-dockerfile mixed-line-ending mypy shellcheck].
           You can pass extra arguments including options to the pre-commit framework as
           <EXTRA_ARGS> passed after --. For example:
 

CONTRIBUTING.rst

@@ -365,6 +365,8 @@ image built locally):
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``check-xml``                         Checks XML files with xmllint.
 ----------------------------------- ---------------------------------------------------------------- ------------
+``debug-statements``                  Detects accidenatally committed debug statements.
+----------------------------------- ---------------------------------------------------------------- ------------
 ``detect-private-key``                Detects if private key is added to the repository.
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``doctoc``                            Refreshes the table of contents for md files.
@@ -385,6 +387,8 @@ image built locally):
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``mypy``                              Runs mypy.                                                           *
 ----------------------------------- ---------------------------------------------------------------- ------------
+``pydevd``                            Check for accidentally commited pydevd statements.
+----------------------------------- ---------------------------------------------------------------- ------------
 ``python-no-log-warn``                Checks if there are no deprecate log warn.
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``rst-backticks``                     Checks if RST files use double backticks for code.

breeze-complete

@@ -22,7 +22,7 @@ _BREEZE_ALLOWED_ENVS=" docker kubernetes "
 _BREEZE_ALLOWED_BACKENDS=" sqlite mysql postgres "
 _BREEZE_ALLOWED_KUBERNETES_VERSIONS=" v1.13.0 "
 _BREEZE_ALLOWED_KUBERNETES_MODES=" persistent_mode git_mode "
-_BREEZE_ALLOWED_STATIC_CHECKS=" all check-hooks-apply check-merge-conflict check-executables-have-shebangs check-xml detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license check-apache-license lint-dockerfile mixed-line-ending mypy shellcheck"
+_BREEZE_ALLOWED_STATIC_CHECKS=" all check-hooks-apply check-merge-conflict check-executables-have-shebangs check-xml debug-statements detect-private-key doctoc end-of-file-fixer flake8 forbid-tabs insert-license check-apache-license lint-dockerfile mixed-line-ending mypy shellcheck"
 _BREEZE_DEFAULT_DOCKERHUB_USER="apache"
 _BREEZE_DEFAULT_DOCKERHUB_REPO="airflow"
 

dev/airflow-license

@@ -32,7 +32,7 @@ _licenses = {'MIT': ['Permission is hereby granted free of charge', 'The above c
 
 def get_notices():
     license_file = open("../LICENSE", "r")
-    regex = ur"\((.+?)\) (.+?) \((http.+?)\)"
+    regex = r"\((.+?)\) (.+?) \((http.+?)\)"
 
     return list(filter(None, [re.findall(regex, line) for line in license_file]))