Requirements now depend on python version (#7841)

(cherry picked from commit 3fb5f15)

.dockerignore

@@ -47,7 +47,7 @@
 !MANIFEST.in
 !NOTICE
 !.github
-!requirements.txt
+!requirements
 
 # Avoid triggering context change on README change (new companies using Airflow)
 # So please do not uncomment this line ;)

.pre-commit-config.yaml

@@ -269,17 +269,10 @@ repos:
         files: \.py$
       - id: build
         name: Check if image build is needed
-        entry: ./scripts/ci/pre_commit_ci_build.sh
+        entry: ./scripts/ci/pre_commit_ci_build.sh 3.5 false
         language: system
         always_run: true
         pass_filenames: false
-      - id: generate-requirements
-        name: Generate requirements
-        entry: "./scripts/ci/pre_commit_generate_requirements.sh"
-        language: system
-        files: ^setup.py$
-        pass_filenames: false
-        require_serial: true
       - id: check-apache-license
         name: Check if licenses are OK for Apache
         entry: "./scripts/ci/pre_commit_check_license.sh"

.rat-excludes

@@ -13,6 +13,7 @@
 .eslintignore
 .flake8
 .rat-excludes
+requirements
 requirements.txt
 .*log
 .travis.yml

.travis.yml

@@ -41,18 +41,17 @@ jobs:
       stage: pre-test
       script: ./scripts/ci/ci_run_all_static_tests.sh
       env: >-
-        PYTHON_VERSION=3.5
+        PYTHON_MAJOR_MINOR_VERSION=3.5
         AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="true"
-        SKIP=generate-requirements
     - name: "Build documentation"
+      env: >-
+        PYTHON_MAJOR_MINOR_VERSION=3.5
       stage: test
       script: ./scripts/ci/ci_docs.sh
-      env: >-
-        PYTHON_VERSION=3.5
     - name: "Tests [Py3.6][Kubernetes][persistent]"
       env: >-
         BACKEND=postgres
-        PYTHON_VERSION=3.6
+        PYTHON_MAJOR_MINOR_VERSION=3.6
         RUNTIME=kubernetes
         ENABLE_KIND_CLUSTER=true
         KUBERNETES_MODE=persistent_mode
@@ -61,7 +60,7 @@ jobs:
     - name: "Tests [Py3.5][Kubernetes][git]"
       env: >-
         BACKEND=postgres
-        PYTHON_VERSION=3.5
+        PYTHON_MAJOR_MINOR_VERSION=3.5
         RUNTIME=kubernetes
         ENABLE_KIND_CLUSTER=true
         KUBERNETES_MODE=git_mode
@@ -70,7 +69,7 @@ jobs:
     - name: "Tests [Py2.7][Kubernetes][persistent]"
       env: >-
         BACKEND=postgres
-        PYTHON_VERSION=2.7
+        PYTHON_MAJOR_MINOR_VERSION=2.7
         ENABLE_KIND_CLUSTER=true
         RUNTIME=kubernetes
         KUBERNETES_MODE=persistent_mode
@@ -79,7 +78,7 @@ jobs:
     - name: "Tests [Py2.7][Kubernetes][git]"
       env: >-
         BACKEND=postgres
-        PYTHON_VERSION=2.7
+        PYTHON_MAJOR_MINOR_VERSION=2.7
         ENABLE_KIND_CLUSTER=true
         RUNTIME=kubernetes
         KUBERNETES_MODE=git_mode
@@ -88,100 +87,115 @@ jobs:
     - name: "Tests [Postgres9.6][Py3.6][integrations]"
       env: >-
         BACKEND=postgres
-        PYTHON_VERSION=3.6
+        PYTHON_MAJOR_MINOR_VERSION=3.6
         POSTGRES_VERSION=9.6
         ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis"
         RUN_INTEGRATION_TESTS=all
       stage: test
     - name: "Tests [Postgres9.6][Py3.6][kerberos]"
       env: >-
         BACKEND=postgres
+        PYTHON_MAJOR_MINOR_VERSION=3.6
         POSTGRES_VERSION=9.6
         ENABLED_INTEGRATIONS="kerberos"
-        PYTHON_VERSION=3.6
       stage: test
     - name: "Tests [Postgres10][Py3.6][integrations]"
       env: >-
         BACKEND=postgres
-        PYTHON_VERSION=3.6
+        PYTHON_MAJOR_MINOR_VERSION=3.6
         POSTGRES_VERSION=10
         ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis"
         RUN_INTEGRATION_TESTS=all
       stage: test
     - name: "Tests [Postgres9.6][Py3.7][kerberos]"
       env: >-
         BACKEND=postgres
+        PYTHON_MAJOR_MINOR_VERSION=3.6
         POSTGRES_VERSION=9.6
         ENABLED_INTEGRATIONS="kerberos"
-        PYTHON_VERSION=3.6
       stage: test
     - name: "Tests [Postgres10][Py3.7][integrations]"
       env: >-
         BACKEND=postgres
-        PYTHON_VERSION=3.6
+        PYTHON_MAJOR_MINOR_VERSION=3.6
         POSTGRES_VERSION=10
         ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis"
         RUN_INTEGRATION_TESTS=all
       stage: test
     - name: "Tests [Postgres10][Py3.6][kerberos]"
       env: >-
         BACKEND=postgres
+        PYTHON_MAJOR_MINOR_VERSION=3.6
         POSTGRES_VERSION=10
         ENABLED_INTEGRATIONS="kerberos"
-        PYTHON_VERSION=3.6
       stage: test
     - name: "Tests [Sqlite][Py2.7][integrations]"
       env: >-
         BACKEND=sqlite
-        PYTHON_VERSION=2.7
+        PYTHON_MAJOR_MINOR_VERSION=2.7
         ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis"
         RUN_INTEGRATION_TESTS=all
       stage: test
     - name: "Tests [Sqlite][Py3.5]"
       env: >-
         BACKEND=sqlite
-        PYTHON_VERSION=3.5
+        PYTHON_MAJOR_MINOR_VERSION=3.5
       stage: test
     - name: "Tests [MySQL5.6][Py3.6][integrations]"
       env: >-
         BACKEND=mysql
-        PYTHON_VERSION=3.6
+        PYTHON_MAJOR_MINOR_VERSION=3.6
         MYSQL_VERSION=5.6
         ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis"
         RUN_INTEGRATION_TESTS=all
       stage: test
     - name: "Tests [MySQL5.6][Py2.7][kerberos]"
       env: >-
         BACKEND=mysql
+        PYTHON_MAJOR_MINOR_VERSION=2.7
         ENABLED_INTEGRATIONS="kerberos"
         MYSQL_VERSION=5.6
-        PYTHON_VERSION=2.7
       stage: test
     - name: "Tests [MySQL5.7][Py3.6][integrations]"
       env: >-
         BACKEND=mysql
-        PYTHON_VERSION=3.6
+        PYTHON_MAJOR_MINOR_VERSION=3.6
         MYSQL_VERSION=5.7
         ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis"
         RUN_INTEGRATION_TESTS=all
       stage: test
     - name: "Tests [MySQL5.7][Py2.7][kerberos]"
       env: >-
         BACKEND=mysql
+        PYTHON_MAJOR_MINOR_VERSION=2.7
+        MYSQL_VERSION=5.7
         ENABLED_INTEGRATIONS="kerberos"
         MYSQL_VERSION=5.7
-        PYTHON_VERSION=2.7
       stage: test
     - name: "Tests [MySQL5.7][Py3.7]"
       env: >-
         BACKEND=mysql
+        PYTHON_MAJOR_MINOR_VERSION=3.7
         MYSQL_VERSION=5.7
-        PYTHON_VERSION=3.7
       stage: test
-    - name: "Generate requirements"
+    - name: "Generate requirements Py2.7"
+      env: >-
+        PYTHON_MAJOR_MINOR_VERSION=2.7
+      stage: test
+      script: ./scripts/ci/ci_generate_requirements.sh
+    - name: "Generate requirements Py3.5"
+      env: >-
+        PYTHON_MAJOR_MINOR_VERSION=3.5
+      stage: test
+      script: ./scripts/ci/ci_generate_requirements.sh
+    - name: "Generate requirements Py3.6"
+      env: >-
+        PYTHON_MAJOR_MINOR_VERSION=3.6
+      stage: test
+      script: ./scripts/ci/ci_generate_requirements.sh
+    - name: "Generate requirements Py3.7"
       env: >-
-        PYTHON_VERSION=3.5
-      before_install: pip install bowler
+        PYTHON_MAJOR_MINOR_VERSION=3.7
       stage: test
       script: ./scripts/ci/ci_generate_requirements.sh
 before_install:

BREEZE.rst

@@ -85,7 +85,7 @@ Docker Images Used by Breeze
 For all development tasks, unit tests, integration tests and static code checks, we use the
 **CI image** maintained on the Docker Hub in the ``apache/airflow`` repository.
 This Docker image contains a lot test-related packages (size of ~1GB).
-Its tag follows the pattern of ``<BRANCH>-python<PYTHON_VERSION>-ci``
+Its tag follows the pattern of ``<BRANCH>-python<PYTHON_MAJOR_MINOR_VERSION>-ci``
 (for example, ``apache/airflow:master-python3.6-ci``). The image is built using the
 `<Dockerfile>`_ Dockerfile.
 
@@ -608,6 +608,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
     build-only                               Only builds docker images without entering container
     cleanup-images                           Cleans up the container images created
     exec                                     Execs into running breeze container in new terminal
+    generate-requirements                    Generates pinned requirements for pip dependencies
     initialize-local-virtualenv              Initializes local virtualenv
     setup-autocomplete                       Sets up autocomplete for breeze
     stop                                     Stops the docker-compose evironment
@@ -679,10 +680,18 @@ This is the current syntax for  `./breeze <./breeze>`_:
         way to run multiple processes in the same container at the same time for example scheduler,
         webserver, workers, database console and interactive terminal.
   ****************************************************************************************************
+  breeze [FLAGS] generate-requirements -- <EXTRA_ARGS>
+
+        Generates pinned requirements from setup.py. Those requirements are generated in requirements
+        directory - separately for different python version. Those requirements are used to run
+        CI builds as well as run repeatable production image builds. You can use those requirements
+        to predictably install released airflow versions. You should run it always after you update
+        setup.py.
+  ****************************************************************************************************
   breeze [FLAGS] initialize-local-virtualenv -- <EXTRA_ARGS>
 
         Initializes locally created virtualenv installing all dependencies of Airflow
-        taking into account the frozen requirements from requirements.txt.
+        taking into account the frozen requirements from requirements folder.
         This local virtualenv can be used to aid autocompletion and IDE support as
         well as run unit tests directly from the IDE. You need to have virtualenv
         activated before running this command.
@@ -805,7 +814,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
    Choose Airflow variant
   ****************************************************************************************************
 
-  -p, --python <PYTHON_VERSION>
+  -p, --python <PYTHON_MAJOR_MINOR_VERSION>
           Python version used for the image. This is always major/minor version.
           One of: