From 8ccb0f5fef25369e8581c565c26bc7770ec2f199 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 27 Mar 2020 15:27:53 +0100 Subject: [PATCH] Requirements now depend on python version (#7841) (cherry picked from commit 3fb5f1568cbd98d49d8a195b65dcaa25539292c6) --- .dockerignore | 2 +- .pre-commit-config.yaml | 9 +- .rat-excludes | 1 + .travis.yml | 62 ++-- BREEZE.rst | 15 +- CONTRIBUTING.rst | 190 ++++++++-- Dockerfile | 27 +- INSTALL | 9 +- breeze | 57 +-- breeze-complete | 5 +- common/_files_for_rebuild_check.sh | 1 - requirements/requirements-python2.7.txt | 331 ++++++++++++++++++ requirements/requirements-python3.5.txt | 315 +++++++++++++++++ .../requirements-python3.6.txt | 13 +- requirements/requirements-python3.7.txt | 313 +++++++++++++++++ scripts/ci/_utils.sh | 56 ++- scripts/ci/ci_before_install.sh | 4 +- scripts/ci/ci_build_dockerhub.sh | 6 +- scripts/ci/ci_check_license.sh | 2 +- scripts/ci/ci_docs.sh | 2 +- scripts/ci/ci_fix_ownership.sh | 4 +- scripts/ci/ci_flake8.sh | 2 +- scripts/ci/ci_generate_requirements.sh | 17 +- scripts/ci/ci_mypy.sh | 2 +- scripts/ci/ci_run_all_static_tests.sh | 2 +- scripts/ci/docker-compose/base.yml | 1 + scripts/ci/docker-compose/local.yml | 2 +- .../ci/in_container/_in_container_utils.sh | 3 +- scripts/ci/in_container/entrypoint_ci.sh | 4 +- .../docker/rebuild_airflow_image.sh | 3 +- scripts/ci/in_container/run_docs_build.sh | 7 +- .../in_container/run_generate_requirements.sh | 39 ++- scripts/ci/pre_commit_ci_build.sh | 4 +- .../ci/pre_commit_generate_requirements.sh | 2 + setup.py | 60 +--- tests/bats/test_local_mounts.bats | 2 +- 36 files changed, 1356 insertions(+), 218 deletions(-) create mode 100644 requirements/requirements-python2.7.txt create mode 100644 requirements/requirements-python3.5.txt rename requirements.txt => requirements/requirements-python3.6.txt (97%) create mode 100644 requirements/requirements-python3.7.txt diff --git a/.dockerignore b/.dockerignore index 7ae1741d2793ae..c694784366a05a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -47,7 +47,7 @@ !MANIFEST.in !NOTICE !.github -!requirements.txt +!requirements # Avoid triggering context change on README change (new companies using Airflow) # So please do not uncomment this line ;) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04be2429dc7bbd..400e73d10e0d21 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -269,17 +269,10 @@ repos: files: \.py$ - id: build name: Check if image build is needed - entry: ./scripts/ci/pre_commit_ci_build.sh + entry: ./scripts/ci/pre_commit_ci_build.sh 3.5 false language: system always_run: true pass_filenames: false - - id: generate-requirements - name: Generate requirements - entry: "./scripts/ci/pre_commit_generate_requirements.sh" - language: system - files: ^setup.py$ - pass_filenames: false - require_serial: true - id: check-apache-license name: Check if licenses are OK for Apache entry: "./scripts/ci/pre_commit_check_license.sh" diff --git a/.rat-excludes b/.rat-excludes index 88b2f5b8473942..906d8044452df8 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -13,6 +13,7 @@ .eslintignore .flake8 .rat-excludes +requirements requirements.txt .*log .travis.yml diff --git a/.travis.yml b/.travis.yml index f12386f3e4993b..37d89280d6e42a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,18 +41,17 @@ jobs: stage: pre-test script: ./scripts/ci/ci_run_all_static_tests.sh env: >- - PYTHON_VERSION=3.5 + PYTHON_MAJOR_MINOR_VERSION=3.5 AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="true" - SKIP=generate-requirements - name: "Build documentation" + env: >- + PYTHON_MAJOR_MINOR_VERSION=3.5 stage: test script: ./scripts/ci/ci_docs.sh - env: >- - PYTHON_VERSION=3.5 - name: "Tests [Py3.6][Kubernetes][persistent]" env: >- BACKEND=postgres - PYTHON_VERSION=3.6 + PYTHON_MAJOR_MINOR_VERSION=3.6 RUNTIME=kubernetes ENABLE_KIND_CLUSTER=true KUBERNETES_MODE=persistent_mode @@ -61,7 +60,7 @@ jobs: - name: "Tests [Py3.5][Kubernetes][git]" env: >- BACKEND=postgres - PYTHON_VERSION=3.5 + PYTHON_MAJOR_MINOR_VERSION=3.5 RUNTIME=kubernetes ENABLE_KIND_CLUSTER=true KUBERNETES_MODE=git_mode @@ -70,7 +69,7 @@ jobs: - name: "Tests [Py2.7][Kubernetes][persistent]" env: >- BACKEND=postgres - PYTHON_VERSION=2.7 + PYTHON_MAJOR_MINOR_VERSION=2.7 ENABLE_KIND_CLUSTER=true RUNTIME=kubernetes KUBERNETES_MODE=persistent_mode @@ -79,7 +78,7 @@ jobs: - name: "Tests [Py2.7][Kubernetes][git]" env: >- BACKEND=postgres - PYTHON_VERSION=2.7 + PYTHON_MAJOR_MINOR_VERSION=2.7 ENABLE_KIND_CLUSTER=true RUNTIME=kubernetes KUBERNETES_MODE=git_mode @@ -88,7 +87,7 @@ jobs: - name: "Tests [Postgres9.6][Py3.6][integrations]" env: >- BACKEND=postgres - PYTHON_VERSION=3.6 + PYTHON_MAJOR_MINOR_VERSION=3.6 POSTGRES_VERSION=9.6 ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis" RUN_INTEGRATION_TESTS=all @@ -96,14 +95,14 @@ jobs: - name: "Tests [Postgres9.6][Py3.6][kerberos]" env: >- BACKEND=postgres + PYTHON_MAJOR_MINOR_VERSION=3.6 POSTGRES_VERSION=9.6 ENABLED_INTEGRATIONS="kerberos" - PYTHON_VERSION=3.6 stage: test - name: "Tests [Postgres10][Py3.6][integrations]" env: >- BACKEND=postgres - PYTHON_VERSION=3.6 + PYTHON_MAJOR_MINOR_VERSION=3.6 POSTGRES_VERSION=10 ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis" RUN_INTEGRATION_TESTS=all @@ -111,14 +110,14 @@ jobs: - name: "Tests [Postgres9.6][Py3.7][kerberos]" env: >- BACKEND=postgres + PYTHON_MAJOR_MINOR_VERSION=3.6 POSTGRES_VERSION=9.6 ENABLED_INTEGRATIONS="kerberos" - PYTHON_VERSION=3.6 stage: test - name: "Tests [Postgres10][Py3.7][integrations]" env: >- BACKEND=postgres - PYTHON_VERSION=3.6 + PYTHON_MAJOR_MINOR_VERSION=3.6 POSTGRES_VERSION=10 ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis" RUN_INTEGRATION_TESTS=all @@ -126,26 +125,26 @@ jobs: - name: "Tests [Postgres10][Py3.6][kerberos]" env: >- BACKEND=postgres + PYTHON_MAJOR_MINOR_VERSION=3.6 POSTGRES_VERSION=10 ENABLED_INTEGRATIONS="kerberos" - PYTHON_VERSION=3.6 stage: test - name: "Tests [Sqlite][Py2.7][integrations]" env: >- BACKEND=sqlite - PYTHON_VERSION=2.7 + PYTHON_MAJOR_MINOR_VERSION=2.7 ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis" RUN_INTEGRATION_TESTS=all stage: test - name: "Tests [Sqlite][Py3.5]" env: >- BACKEND=sqlite - PYTHON_VERSION=3.5 + PYTHON_MAJOR_MINOR_VERSION=3.5 stage: test - name: "Tests [MySQL5.6][Py3.6][integrations]" env: >- BACKEND=mysql - PYTHON_VERSION=3.6 + PYTHON_MAJOR_MINOR_VERSION=3.6 MYSQL_VERSION=5.6 ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis" RUN_INTEGRATION_TESTS=all @@ -153,14 +152,14 @@ jobs: - name: "Tests [MySQL5.6][Py2.7][kerberos]" env: >- BACKEND=mysql + PYTHON_MAJOR_MINOR_VERSION=2.7 ENABLED_INTEGRATIONS="kerberos" MYSQL_VERSION=5.6 - PYTHON_VERSION=2.7 stage: test - name: "Tests [MySQL5.7][Py3.6][integrations]" env: >- BACKEND=mysql - PYTHON_VERSION=3.6 + PYTHON_MAJOR_MINOR_VERSION=3.6 MYSQL_VERSION=5.7 ENABLED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis" RUN_INTEGRATION_TESTS=all @@ -168,20 +167,35 @@ jobs: - name: "Tests [MySQL5.7][Py2.7][kerberos]" env: >- BACKEND=mysql + PYTHON_MAJOR_MINOR_VERSION=2.7 + MYSQL_VERSION=5.7 ENABLED_INTEGRATIONS="kerberos" MYSQL_VERSION=5.7 - PYTHON_VERSION=2.7 stage: test - name: "Tests [MySQL5.7][Py3.7]" env: >- BACKEND=mysql + PYTHON_MAJOR_MINOR_VERSION=3.7 MYSQL_VERSION=5.7 - PYTHON_VERSION=3.7 stage: test - - name: "Generate requirements" + - name: "Generate requirements Py2.7" + env: >- + PYTHON_MAJOR_MINOR_VERSION=2.7 + stage: test + script: ./scripts/ci/ci_generate_requirements.sh + - name: "Generate requirements Py3.5" + env: >- + PYTHON_MAJOR_MINOR_VERSION=3.5 + stage: test + script: ./scripts/ci/ci_generate_requirements.sh + - name: "Generate requirements Py3.6" + env: >- + PYTHON_MAJOR_MINOR_VERSION=3.6 + stage: test + script: ./scripts/ci/ci_generate_requirements.sh + - name: "Generate requirements Py3.7" env: >- - PYTHON_VERSION=3.5 - before_install: pip install bowler + PYTHON_MAJOR_MINOR_VERSION=3.7 stage: test script: ./scripts/ci/ci_generate_requirements.sh before_install: diff --git a/BREEZE.rst b/BREEZE.rst index 9e0dd175aa31d6..74083da917076d 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -85,7 +85,7 @@ Docker Images Used by Breeze For all development tasks, unit tests, integration tests and static code checks, we use the **CI image** maintained on the Docker Hub in the ``apache/airflow`` repository. This Docker image contains a lot test-related packages (size of ~1GB). -Its tag follows the pattern of ``-python-ci`` +Its tag follows the pattern of ``-python-ci`` (for example, ``apache/airflow:master-python3.6-ci``). The image is built using the ``_ Dockerfile. @@ -608,6 +608,7 @@ This is the current syntax for `./breeze <./breeze>`_: build-only Only builds docker images without entering container cleanup-images Cleans up the container images created exec Execs into running breeze container in new terminal + generate-requirements Generates pinned requirements for pip dependencies initialize-local-virtualenv Initializes local virtualenv setup-autocomplete Sets up autocomplete for breeze stop Stops the docker-compose evironment @@ -679,10 +680,18 @@ This is the current syntax for `./breeze <./breeze>`_: way to run multiple processes in the same container at the same time for example scheduler, webserver, workers, database console and interactive terminal. **************************************************************************************************** + breeze [FLAGS] generate-requirements -- + + Generates pinned requirements from setup.py. Those requirements are generated in requirements + directory - separately for different python version. Those requirements are used to run + CI builds as well as run repeatable production image builds. You can use those requirements + to predictably install released airflow versions. You should run it always after you update + setup.py. + **************************************************************************************************** breeze [FLAGS] initialize-local-virtualenv -- Initializes locally created virtualenv installing all dependencies of Airflow - taking into account the frozen requirements from requirements.txt. + taking into account the frozen requirements from requirements folder. This local virtualenv can be used to aid autocompletion and IDE support as well as run unit tests directly from the IDE. You need to have virtualenv activated before running this command. @@ -805,7 +814,7 @@ This is the current syntax for `./breeze <./breeze>`_: Choose Airflow variant **************************************************************************************************** - -p, --python + -p, --python Python version used for the image. This is always major/minor version. One of: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index cc3b3477bb63f5..ec0a24760645ee 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -327,8 +327,8 @@ webhdfs, winrm .. END EXTRAS HERE -Pinned Airflow requirements.txt file ------------------------------------- +Airflow dependencies +-------------------- Airflow is not a standard python project. Most of the python projects fall into one of two types - application or library. As described in @@ -343,30 +343,134 @@ be open to allow several different libraries with the same requirements to be in The problem is that Apache Airflow is a bit of both - application to install and library to be used when you are developing your own operators and DAGs. -This - seemingly unsolvable - puzzle is solved as follows: +This - seemingly unsolvable - puzzle is solved by having pinned requirement files. Those are available +as of airflow 1.10.10. -* by default when you install ``apache-airflow`` package - the dependencies are as open as possible while - still allowing the apache-airflow to install. This means that 'apache-airflow' package might fail to - install in case a direct or transitive dependency is released that breaks the installation. In such case - when installing ``apache-airflow``, you might need to provide additional constraints (for - example ``pip install apache-airflow==1.10.2 Werkzeug<1.0.0``) +Pinned requirement files +------------------------ -* we have ``requirements.txt`` file generated automatically based on the set of all latest working - and tested requirement versions. You can also use that file as a constraints file when installing - apache airflow - either from the sources ``pip install -e . --constraint requirements.txt`` or - from the pypi package ``pip install apache-airflow --constraint requirements.txt``. Note that - this will also work with extras for example ``pip install .[gcp] --constraint requirements.txt`` or - ``pip install apache-airflow[gcp] --constraint requirements.txt`` +By default when you install ``apache-airflow`` package - the dependencies are as open as possible while +still allowing the apache-airflow package to install. This means that 'apache-airflow' package might fail to +install in case a direct or transitive dependency is released that breaks the installation. In such case +when installing ``apache-airflow``, you might need to provide additional constraints (for +example ``pip install apache-airflow==1.10.2 Werkzeug<1.0.0``) -The ``requirements.txt`` file should be updated automatically via pre-commit whenever you update dependencies -It reflects the current set of dependencies installed in the CI image of Apache Airflow. -The same set of requirements will be used to produce the production image. +However we now have ``requirements-python.txt`` file generated +automatically and committed in the requirements folder based on the set of all latest working and tested +requirement versions. Those ``requirement-python.txt`` files can be used as +constraints file when installing Apache Airflow - either from the sources -If you do not use pre-commits and the CI builds fails / you need to regenerate it, you can do it manually: -``pre-commit run generate-requirements --all-files`` or via script -``./scripts/ci/ci_generate_requirements.sh``. -This will try to regenerate the requirements.txt file with the latest requirements matching -the setup.py constraints. +.. code-block:: bash + + pip install -e . --constraint requirements/requirements-python3.6.txt + + +or from the pypi package + +.. code-block:: bash + + pip install apache-airflow --constraint requirements/requirements-python3.6.txt + + +This works also with extras - for example: + +.. code-block:: bash + + pip install .[gcp] --constraint requirements/requirements-python3.6.txt + + +It is also possible to use constraints directly from github using tag/version name: + +.. code-block:: bash + + pip install apache-airflow[gcp]==1.10.10 \ + --constraint https://raw.githubusercontent.com/apache/airflow/1.10.10/requirements/requirements-python3.6.txt + +There are different set of fixed requirements for different python major/minor versions and you should +use the right requirements file for the right python version. + +The ``requirements-python.txt`` file MUST be regenerated every time after +the ``setup.py`` is updated. This is checked automatically in Travis CI build. There are separate +jobs for each python version that checks if the requirements should be updated. + +If they are not updated, you should regenerate the requirements locally using Breeze as described below. + +Generating requirement files +---------------------------- + +This should be done every time after you modify setup.py file. You can generate requirement files +using `Breeze `_ . Simply use those commands: + +.. code-block:: bash + + breeze generate-requirements --python 3.7 + +.. code-block:: bash + + breeze generate-requirements --python 3.6 + +Note that when you generate requirements this way, you might update to latest version of requirements +that were released since the last time so during tests you might get errors unrelated to your change. +In this case the easiest way to fix it is to limit the culprit dependency to the previous version +with ```` constraint added in setup.py. + +Backport providers packages +--------------------------- + +Since we are developing new operators in the master branch, we prepared backport packages ready to be +installed for Airflow 1.10.* series. Those backport operators (the tested ones) are going to be released +in PyPi and we are going to maintain the list at +`Backported providers package page `_ + +Some of the packages have cross-dependencies with other providers packages. This typically happens for +transfer operators where operators use hooks from the other providers in case they are transferring +data between the providers. The list of dependencies is maintained (automatically with pre-commits) +in the ``airflow/providers/dependencies.json``. Pre-commits are also used to generate dependencies. +The dependency list is automatically used during pypi packages generation. + +Cross-dependencies between provider packages are converted into extras - if you need functionality from +the other provider package you can install it adding [extra] after the apache-airflow-providers-PROVIDER +for example ``pip install apache-airflow-providers-google[amazon]`` in case you want to use GCP's +transfer operators from Amazon ECS. + +If you add a new dependency between different providers packages, it will be detected automatically during +pre-commit phase and pre-commit will fail - and add entry in dependencies.json so that the package extra +dependencies are properly added when package is installed. + +You can regenerate the whole list of provider dependencies by running this command (you need to have +``pre-commits`` installed). + +.. code-block:: bash + + pre-commit run build-providers-dependencies + + +Here is the list of packages and their extras: + + + .. START PACKAGE DEPENDENCIES HERE + +========================== =========================== +Package Extras +========================== =========================== +amazon apache.hive,google,imap,mongo,postgres,ssh +apache.druid apache.hive +apache.hive amazon,microsoft.mssql,mysql,presto,samba,vertica +apache.livy http +dingding http +discord http +google amazon,apache.cassandra,cncf.kubernetes,microsoft.azure,microsoft.mssql,mysql,postgres,presto,sftp +hashicorp google +microsoft.azure oracle +microsoft.mssql odbc +mysql amazon,presto,vertica +opsgenie http +postgres amazon +sftp ssh +slack http +========================== =========================== + + .. END PACKAGE DEPENDENCIES HERE Static code checks ================== @@ -378,6 +482,48 @@ Your code must pass all the static code checks in Travis CI in order to be eligi The easiest way to make sure your code is good before pushing is to use pre-commit checks locally as described in the static code checks documentation. +.. _coding_style: + +Coding style and best practices +=============================== + +Most of our coding style rules are enforced programmatically by flake8 and pylint (which are run automatically +on every pull request), but there are some rules that are not yet automated and are more Airflow specific or +semantic than style + +Database Session Handling +------------------------- + +**Explicit is better than implicit.** If a function accepts a ``session`` parameter it should not commit the +transaction itself. Session management is up to the caller. + +To make this easier there is the ``create_session`` helper: + +.. code-block:: python + + from airflow.utils.session import create_session + + def my_call(*args, session): + ... + # You MUST not commit the session here. + + with create_session() as session: + my_call(*args, session=session) + +If this function is designed to be called by "end-users" (i.e. DAG authors) then using the ``@provide_session`` wrapper is okay: + +.. code-block:: python + + from airflow.utils.session import provide_session + + ... + + @provide_session + def my_method(arg, arg, session=None) + ... + # You SHOULD not commit the session here. The wrapper will take care of commit()/rollback() if exception + + Test Infrastructure =================== diff --git a/Dockerfile b/Dockerfile index 42248d2d0c60ec..fda92074d4447b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,12 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} ARG AIRFLOW_VERSION="2.0.0.dev0" ENV AIRFLOW_VERSION=$AIRFLOW_VERSION +ARG PYTHON_MAJOR_MINOR_VERSION="3.6" +ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION} + +ARG UPGRADE_TO_LATEST_REQUIREMENTS="false" +ENV UPGRADE_TO_LATEST_REQUIREMENTS=${UPGRADE_TO_LATEST_REQUIREMENTS} + # Print versions RUN echo "Base image: ${PYTHON_BASE_IMAGE}" RUN echo "Airflow version: ${AIRFLOW_VERSION}" @@ -126,12 +132,14 @@ RUN mkdir -pv /usr/share/man/man1 \ krb5-user \ ldap-utils \ less \ + # The latest buster images do not have libpython 2.7 installed and it is needed + # To run virtualenv tests with python 2 + libpython2.7-stdlib \ lsb-release \ net-tools \ openssh-client \ openssh-server \ postgresql-client \ - python-selinux \ sqlite3 \ tmux \ unzip \ @@ -220,7 +228,7 @@ RUN echo "Pip version: ${PIP_VERSION}" RUN pip install --upgrade pip==${PIP_VERSION} # Install Google SDK -ENV GCLOUD_HOME="/opt/gcloud" +ENV GCLOUD_HOME="/opt/gcloud" CLOUDSDK_PYTHON=python${PYTHON_MAJOR_MINOR_VERSION} RUN GCLOUD_VERSION="274.0.1" \ && GCOUD_URL="https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz" \ @@ -289,7 +297,7 @@ ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS} RUN echo "Installing with extras: ${AIRFLOW_EXTRAS}." -ARG AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="false" +ARG AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="true" ENV AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD=${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD} # By changing the CI build epoch we can force reinstalling Arflow from the current master @@ -304,7 +312,7 @@ ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} RUN \ if [[ "${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" == "true" ]]; then \ pip install \ - "https://github.com/apache/airflow/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ + "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ && pip uninstall --yes apache-airflow; \ fi @@ -331,19 +339,20 @@ COPY airflow/version.py ${AIRFLOW_SOURCES}/airflow/version.py COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py COPY airflow/bin/airflow ${AIRFLOW_SOURCES}/airflow/bin/airflow -COPY requirements.txt ${AIRFLOW_SOURCES}/requirements.txt +COPY requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt \ + ${AIRFLOW_SOURCES}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt -ENV UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD=${UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD} # The goal of this line is to install the dependencies from the most current setup.py from sources # This will be usually incremental small set of packages in CI optimized build, so it will be very fast # In non-CI optimized build this will install all dependencies before installing sources. -# Usually we will install versions constrained to the current requirements.txt +# Usually we will install versions constrained to the current requirements file # But in cron job we will install latest versions matching setup.py to see if there is no breaking change RUN \ - if [[ "${UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD}" == "true" ]]; then \ + if [[ "${UPGRADE_TO_LATEST_REQUIREMENTS}" == "true" ]]; then \ pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade; \ else \ - pip install -e ".[${AIRFLOW_EXTRAS}]" --constraint ${AIRFLOW_SOURCES}/requirements.txt ; \ + pip install -e ".[${AIRFLOW_EXTRAS}]" \ + --constraint ${AIRFLOW_SOURCES}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt ; \ fi # Copy all the www/ files we need to compile assets. Done as two separate COPY diff --git a/INSTALL b/INSTALL index 149f1da2e5a728..1392312d4ec66a 100644 --- a/INSTALL +++ b/INSTALL @@ -34,11 +34,12 @@ pip install . # or directly python setup.py install -# You can also install recommended version of the dependencies by using requirements.txt -# as constraint file. This is needed in case you have problems with installint latest -# requirements. +# You can also install recommended version of the dependencies by using +# requirements-python.txt as constraint file. This is needed in case +# you have problems with installing the current requirements from PyPI. +# There are different requirements for different python versions. For example" -pip install . --constraint requirements.txt +pip install . --constraint requirements/requirements-python3.7.txt # You can also install Airflow with extras specified. The list of available extras: # START EXTRAS HERE diff --git a/breeze b/breeze index b133b00cef3299..6eadb8ebe38663 100755 --- a/breeze +++ b/breeze @@ -52,7 +52,7 @@ function setup_default_breeze_variables() { mkdir -pv "${FILES_DIR}" # Note - we do not use __script_init.sh here because it can only be used from within - # the CI directory and we need to overrride PYTHON_VERSION based on what we store + # the CI directory and we need to overrride PYTHON_MAJOR_MINOR_VERSION based on what we store # in the .build directory # Beginning of the initialisation here @@ -60,7 +60,7 @@ function setup_default_breeze_variables() { # shellcheck source=scripts/ci/_utils.sh . "${SCRIPTS_CI_DIR}/_utils.sh" - export PYTHON_VERSION="${PYTHON_VERSION:=$(read_from_file PYTHON_VERSION)}" + export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(read_from_file PYTHON_MAJOR_MINOR_VERSION)}" if [[ ${FORCE_SCREEN_WIDTH:="false"} != "true" ]]; then # Sets width of the screen from terminal @@ -148,14 +148,14 @@ function setup_default_breeze_variables() { _BREEZE_DEFAULT_POSTGRES_VERSION="9.6" _BREEZE_DEFAULT_MYSQL_VERSION="5.7" - STATIC_CHECK_PYTHON_VERSION=3.6 + STATIC_CHECK_PYTHON_MAJOR_MINOR_VERSION=3.6 } # End of initialisation here function initialize_virtualenv() { # Check if we are in virtualenv set +e - echo -e "import sys\nif not hasattr(sys,'base_prefix'):\n sys.exit(1)" | "python${PYTHON_VERSION}" + echo -e "import sys\nif not hasattr(sys,'base_prefix'):\n sys.exit(1)" | "python${PYTHON_MAJOR_MINOR_VERSION}" RES=$? set -e if [[ ${RES} != "0" ]]; then @@ -178,7 +178,7 @@ function initialize_virtualenv() { echo pushd "${MY_DIR}" set +e - pip install -e ".[devel]" --constraint requirements.txt + pip install -e ".[devel]" --constraint "requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt" RES=$? set -e popd @@ -334,7 +334,7 @@ function print_badge { Docker image: ${AIRFLOW_CI_IMAGE} Airflow source version: ${AIRFLOW_VERSION} Airflow installed: ${INSTALL_AIRFLOW_VERSION} - Python version: ${PYTHON_VERSION} + Python version: ${PYTHON_MAJOR_MINOR_VERSION} DockerHub user: ${DOCKERHUB_USER} DockerHub repo: ${DOCKERHUB_REPO} Backend: ${BACKEND} ${BACKEND_VERSION} @@ -358,7 +358,7 @@ EOF Docker image: ${AIRFLOW_CI_IMAGE} Airflow source version: ${AIRFLOW_VERSION} Airflow installed: ${INSTALL_AIRFLOW_VERSION} - Python version: ${PYTHON_VERSION} + Python version: ${PYTHON_MAJOR_MINOR_VERSION} DockerHub user: ${DOCKERHUB_USER} DockerHub repo: ${DOCKERHUB_REPO} Backend: ${BACKEND} ${BACKEND_VERSION} @@ -389,7 +389,7 @@ cd "\$(pwd)" || exit export DOCKERHUB_USER=${DOCKERHUB_USER} export DOCKERHUB_REPO=${DOCKERHUB_REPO} export COMPOSE_FILE="${COMPOSE_FILE}" -export PYTHON_VERSION="${PYTHON_VERSION}" +export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" export BACKEND="${BACKEND}" export RUNTIME="${RUNTIME}" export ENABLE_KIND_CLUSTER="${ENABLE_KIND_CLUSTER}" @@ -451,9 +451,9 @@ function prepare_command_files() { CI_ENTRYPOINT_FILE="/opt/airflow/scripts/ci/in_container/entrypoint_ci.sh" # Base python image for the build - export PYTHON_BASE_IMAGE=python:${PYTHON_VERSION}-slim-buster - export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_VERSION}-ci" - export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_VERSION}" + export PYTHON_BASE_IMAGE=python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster + export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" + export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}" DC_RUN_COMMAND="run --service-ports --rm airflow-testing \"${CI_ENTRYPOINT_FILE} " @@ -522,8 +522,8 @@ function parse_arguments() { RUN_HELP="true" shift ;; -p|--python) - export PYTHON_VERSION="${2}"; - echo "Python version: ${PYTHON_VERSION}" + export PYTHON_MAJOR_MINOR_VERSION="${2}"; + echo "Python version: ${PYTHON_MAJOR_MINOR_VERSION}" echo shift 2 ;; -b|--backend) @@ -727,6 +727,10 @@ function parse_arguments() { LAST_SUBCOMMAND="${1}" COMMAND_TO_RUN="run_in_bash" shift ;; + generate-requirements) + LAST_SUBCOMMAND="${1}" + COMMAND_TO_RUN="perform_generate_requirements" + shift ;; initialize-local-virtualenv) LAST_SUBCOMMAND="${1}" echo "Initializing local virtualenv" @@ -750,7 +754,7 @@ function parse_arguments() { echo "${CMDNAME} static-check mypy" exit 1 fi - export PYTHON_VERSION=${STATIC_CHECK_PYTHON_VERSION} + export PYTHON_MAJOR_MINOR_VERSION=${STATIC_CHECK_PYTHON_MAJOR_MINOR_VERSION} export STATIC_CHECK="${2}" export STATIC_CHECK_ALL_FILES="false" EXTRA_STATIC_CHECK_OPTIONS+=("--show-diff-on-failure") @@ -766,7 +770,7 @@ function parse_arguments() { exit 1 fi COMMAND_TO_RUN="perform_static_checks" - export PYTHON_VERSION=${STATIC_CHECK_PYTHON_VERSION} + export PYTHON_MAJOR_MINOR_VERSION=${STATIC_CHECK_PYTHON_MAJOR_MINOR_VERSION} export STATIC_CHECK="${2}" export STATIC_CHECK_ALL_FILES="true" EXTRA_STATIC_CHECK_OPTIONS+=("--all-files" "--show-diff-on-failure") @@ -854,7 +858,7 @@ prepare_allowed_versions() { INDENT=15 LIST_PREFIX=$(printf "%-${INDENT}s" " ") WIDTH=$((SCREEN_WIDTH - INDENT)) - ALLOWED_PYTHON_VERSIONS=$(echo "${_BREEZE_ALLOWED_PYTHON_VERSIONS=""}" | tr '\n' ' ' | \ + ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS=$(echo "${_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS=""}" | tr '\n' ' ' | \ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/") ALLOWED_BACKENDS=$(echo "${_BREEZE_ALLOWED_BACKENDS=""}" | tr '\n' ' ' | \ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/") @@ -884,6 +888,7 @@ prepare_usage() { export USAGE_DOCKER_COMPOSE="Executes specified docker-compose command" export USAGE_EXECUTE_COMMAND="Executes specified command in the container" export USAGE_FLAGS="Shows all breeze's flags" + export USAGE_GENERATE_REQUIREMENTS="Generates pinned requirements for pip dependencies" export USAGE_INITIALIZE_LOCAL_VIRTUALENV="Initializes local virtualenv" export USAGE_SETUP_AUTOCOMPLETE="Sets up autocomplete for breeze" export USAGE_STOP="Stops the docker-compose evironment" @@ -954,10 +959,17 @@ prepare_usage() { " export DETAILED_USAGE_FLAGS=" Explains in detail all the flags that can be used with breeze. +" + export DETAILED_USAGE_GENERATE_REQUIREMENTS=" + Generates pinned requirements from setup.py. Those requirements are generated in requirements + directory - separately for different python version. Those requirements are used to run + CI builds as well as run repeatable production image builds. You can use those requirements + to predictably install released airflow versions. You should run it always after you update + setup.py. " export DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV=" Initializes locally created virtualenv installing all dependencies of Airflow - taking into account the frozen requirements from requirements.txt. + taking into account the frozen requirements from requirements folder. This local virtualenv can be used to aid autocompletion and IDE support as well as run unit tests directly from the IDE. You need to have virtualenv activated before running this command. @@ -1101,11 +1113,11 @@ $(print_star_line) Choose Airflow variant $(print_star_line) --p, --python +-p, --python Python version used for the image. This is always major/minor version. One of: -${ALLOWED_PYTHON_VERSIONS} +${ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS} -b, --backend Backend to use for tests - it determines which database is used. @@ -1331,7 +1343,7 @@ function read_saved_environment_variables { } function check_and_save_all_params() { - check_and_save_allowed_param "PYTHON_VERSION" "Python version" "--python" + check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python" check_and_save_allowed_param "BACKEND" "backend" "--backend" check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode" check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version" @@ -1489,7 +1501,7 @@ function run_build_command { enter_breeze|build_docs|run_tests|run_docker_compose|run_in_bash) rebuild_ci_image_if_needed ;; - perform_static_checks|build_ci_images_only) + perform_static_checks|build_ci_images_only|perform_generate_requirements) rebuild_ci_image_if_needed ;; cleanup_images|run_exec) @@ -1548,6 +1560,9 @@ function run_breeze_command { cleanup_images) remove_images ;; + perform_generate_requirements) + run_generate_requirements + ;; perform_initialize_local_virtualenv) initialize_virtualenv ;; diff --git a/breeze-complete b/breeze-complete index 7ac52ab127bd2e..be47b31bd74992 100644 --- a/breeze-complete +++ b/breeze-complete @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -_BREEZE_ALLOWED_PYTHON_VERSIONS="2.7 3.5 3.6 3.7" +_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS="2.7 3.5 3.6 3.7" _BREEZE_ALLOWED_BACKENDS="sqlite mysql postgres" _BREEZE_ALLOWED_INTEGRATIONS="cassandra kerberos mongo openldap rabbitmq redis all" _BREEZE_ALLOWED_KUBERNETES_MODES="persistent_mode git_mode" @@ -76,6 +76,7 @@ build-docs build-only cleanup-images exec +generate-requirements initialize-local-virtualenv setup-autocomplete stop @@ -104,7 +105,7 @@ _BREEZE_KNOWN_VALUES="" function _get_known_values_breeze() { case "$1" in -p | --python) - _BREEZE_KNOWN_VALUES=${_BREEZE_ALLOWED_PYTHON_VERSIONS} + _BREEZE_KNOWN_VALUES=${_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS} ;; -b | --backend) _BREEZE_KNOWN_VALUES=${_BREEZE_ALLOWED_BACKENDS} diff --git a/common/_files_for_rebuild_check.sh b/common/_files_for_rebuild_check.sh index 80e7e746636b47..74d2adda25328f 100644 --- a/common/_files_for_rebuild_check.sh +++ b/common/_files_for_rebuild_check.sh @@ -20,7 +20,6 @@ FILES_FOR_REBUILD_CHECK=( "setup.py" "setup.cfg" - "requirements.txt" "Dockerfile" ".dockerignore" "airflow/version.py" diff --git a/requirements/requirements-python2.7.txt b/requirements/requirements-python2.7.txt new file mode 100644 index 00000000000000..39984082b43d99 --- /dev/null +++ b/requirements/requirements-python2.7.txt @@ -0,0 +1,331 @@ +adal==1.2.2 +alabaster==0.7.12 +alembic==1.4.2 +amqp==2.5.2 +analytics-python==1.2.9 +ansiwrap==0.8.4 +# Editable install with no version control (apache-airflow==1.10.9) +apispec==2.0.2 +appdirs==1.4.3 +argcomplete==1.11.1 +asn1crypto==1.3.0 +aspy.yaml==1.3.0 +astroid==1.6.6 +atlasclient==1.0.0 +atomicwrites==1.3.0 +attrs==19.3.0 +aws-sam-translator==1.22.0 +aws-xray-sdk==2.4.3 +azure-common==1.1.25 +azure-cosmos==3.1.2 +azure-datalake-store==0.0.48 +azure-mgmt-containerinstance==1.5.0 +azure-mgmt-nspkg==3.0.2 +azure-mgmt-resource==8.0.1 +azure-nspkg==3.0.2 +azure-storage==0.36.0 +azure-storage-blob==2.1.0 +azure-storage-common==2.1.0 +azure-storage-nspkg==3.1.0 +Babel==2.8.0 +backports-abc==0.5 +backports.functools-lru-cache==1.6.1 +backports.shutil-get-terminal-size==1.0.0 +backports.ssl-match-hostname==3.7.0.1 +backports.tempfile==1.0 +backports.weakref==1.0.post1 +bcrypt==3.1.7 +beautifulsoup4==4.7.1 +billiard==3.6.3.0 +bleach==3.1.4 +blinker==1.4 +boto==2.49.0 +boto3==1.12.31 +botocore==1.15.31 +cached-property==1.5.1 +cachetools==3.1.1 +cassandra-driver==3.20.2 +cattrs==0.9.0 +celery==4.4.2 +certifi==2019.11.28 +cffi==1.14.0 +cfgv==2.0.1 +cfn-lint==0.29.2 +cgroupspy==0.1.6 +chardet==3.0.4 +click==6.7 +cloudant==0.5.10 +colorama==0.4.3 +colorlog==4.0.2 +configparser==3.5.3 +contextdecorator==0.10.0 +contextlib2==0.6.0.post1 +cookies==2.2.1 +coverage==5.0.4 +croniter==0.3.31 +cryptography==2.8 +cx-Oracle==7.3.0 +datadog==0.35.0 +decorator==4.4.2 +defusedxml==0.6.0 +dill==0.3.1.1 +distlib==0.3.0 +dnspython==1.16.0 +docker==3.7.3 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.16 +ecdsa==0.15 +elasticsearch==5.5.3 +elasticsearch-dsl==5.4.0 +entrypoints==0.3 +enum34==1.1.10 +fastavro==0.23.0 +filelock==3.0.12 +flake8==3.7.9 +flake8-colors==0.1.6 +flaky==3.6.1 +Flask==1.1.1 +Flask-Admin==1.5.4 +Flask-AppBuilder==1.13.1 +Flask-Babel==0.12.2 +Flask-Bcrypt==0.7.1 +Flask-Caching==1.3.3 +Flask-JWT-Extended==3.24.1 +Flask-Login==0.4.1 +Flask-OpenID==1.2.5 +Flask-SQLAlchemy==2.4.1 +flask-swagger==0.2.13 +Flask-WTF==0.14.3 +flower==0.9.3 +freezegun==0.3.15 +funcsigs==1.0.2 +functools32==3.2.3.post2 +future==0.18.2 +future-fstrings==1.2.0 +futures==3.3.0 +gcsfs==0.2.3 +google-api-core==1.16.0 +google-api-python-client==1.8.0 +google-auth==1.12.0 +google-auth-httplib2==0.0.3 +google-auth-oauthlib==0.4.1 +google-cloud-bigquery==1.24.0 +google-cloud-bigtable==1.2.1 +google-cloud-container==0.4.0 +google-cloud-core==1.3.0 +google-cloud-dlp==0.13.0 +google-cloud-language==1.3.0 +google-cloud-secret-manager==0.2.0 +google-cloud-spanner==1.15.0 +google-cloud-speech==1.3.2 +google-cloud-storage==1.26.0 +google-cloud-texttospeech==1.0.1 +google-cloud-translate==2.0.1 +google-cloud-videointelligence==1.14.0 +google-cloud-vision==1.0.0 +google-resumable-media==0.5.0 +googleapis-common-protos==1.51.0 +graphviz==0.13.2 +grpc-google-iam-v1==0.12.3 +grpcio==1.27.2 +grpcio-gcp==0.2.2 +gunicorn==19.10.0 +hdfs==2.5.8 +hmsclient==0.1.1 +httplib2==0.17.0 +hvac==0.10.0 +identify==1.4.13 +idna==2.9 +ijson==2.6.1 +imagesize==1.2.0 +importlib-metadata==1.6.0 +importlib-resources==1.4.0 +inflection==0.3.1 +ipaddress==1.0.23 +ipdb==0.13.2 +ipython==5.9.0 +ipython-genutils==0.2.0 +iso8601==0.1.12 +isodate==0.6.0 +itsdangerous==1.1.0 +JayDeBeApi==1.1.1 +Jinja2==2.10.3 +jira==2.0.0 +jmespath==0.9.5 +JPype1==0.7.1 +json-merge-patch==0.2 +jsondiff==1.1.2 +jsonpatch==1.25 +jsonpickle==1.3 +jsonpointer==2.0 +jsonschema==3.2.0 +jupyter-client==5.3.4 +jupyter-core==4.6.3 +kombu==4.6.3 +kubernetes==11.0.0 +lazy-object-proxy==1.4.3 +ldap3==2.7 +linecache2==1.0.0 +lockfile==0.12.2 +Mako==1.1.2 +Markdown==2.6.11 +MarkupSafe==1.1.1 +marshmallow==2.19.5 +marshmallow-enum==1.5.1 +marshmallow-sqlalchemy==0.18.0 +mccabe==0.6.1 +mistune==0.8.4 +mock==3.0.5 +mongomock==3.19.0 +monotonic==1.5 +more-itertools==5.0.0 +moto==1.3.14 +msrest==0.6.11 +msrestazure==0.6.3 +multi-key-dict==2.0.3 +mysqlclient==1.3.14 +nbconvert==5.6.1 +nbformat==4.4.0 +networkx==2.2 +nodeenv==1.3.5 +nteract-scrapbook==0.3.1 +ntlm-auth==1.4.0 +numpy==1.16.6 +oauthlib==3.1.0 +oscrypto==1.2.0 +packaging==20.3 +pandas==0.24.2 +pandas-gbq==0.13.1 +pandocfilters==1.4.2 +papermill==1.2.1 +parameterized==0.7.1 +paramiko==2.7.1 +pathlib2==2.3.5 +pathspec==0.7.0 +pbr==5.4.4 +pendulum==1.4.4 +pexpect==4.8.0 +pickleshare==0.7.5 +pinotdb==0.1.1 +pluggy==0.13.1 +pre-commit==1.21.0 +presto-python-client==0.7.0 +prison==0.1.0 +prompt-toolkit==1.0.18 +protobuf==3.11.3 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycodestyle==2.5.0 +pycparser==2.20 +pycryptodomex==3.9.7 +pydata-google-auth==0.3.0 +pydruid==0.5.8 +pyflakes==2.1.1 +Pygments==2.5.2 +PyHive==0.6.2 +PyJWT==1.7.1 +pykerberos==1.2.1 +pymongo==3.10.1 +pymssql==2.1.4 +PyNaCl==1.3.0 +pyOpenSSL==19.1.0 +pyparsing==2.4.6 +pyrsistent==0.16.0 +pysftp==0.2.9 +PySmbClient==0.1.5 +pytest==4.6.9 +pytest-cov==2.8.1 +pytest-instafail==0.4.1.post0 +python-daemon==2.1.2 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-http-client==3.2.6 +python-jenkins==1.7.0 +python-jose==3.1.0 +python-openid==2.2.5 +pytz==2019.3 +pytzdata==2019.3 +pywinrm==0.4.1 +PyYAML==5.3.1 +pyzmq==19.0.0 +qds-sdk==1.15.0 +redis==3.4.1 +requests==2.23.0 +requests-futures==0.9.4 +requests-kerberos==0.12.0 +requests-mock==1.7.0 +requests-ntlm==1.1.0 +requests-oauthlib==1.3.0 +requests-toolbelt==0.9.1 +responses==0.10.12 +rsa==4.0 +s3transfer==0.3.3 +sasl==0.2.1 +scandir==1.10.0 +sendgrid==5.6.0 +sentinels==1.0.0 +sentry-sdk==0.14.3 +setproctitle==1.1.10 +simplegeneric==0.8.1 +singledispatch==3.4.0.3 +six==1.14.0 +slackclient==1.3.2 +snakebite==2.11.0 +snowballstemmer==2.0.0 +snowflake-connector-python==2.1.3 +snowflake-sqlalchemy==1.2.2 +soupsieve==1.9.5 +Sphinx==1.8.5 +sphinx-argparse==0.2.5 +sphinx-autoapi==1.0.0 +sphinx-jinja==1.1.1 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-dotnetdomain==0.4 +sphinxcontrib-golangdomain==0.2.0.dev0 +sphinxcontrib-httpdomain==1.7.0 +sphinxcontrib-websupport==1.1.2 +SQLAlchemy==1.3.15 +SQLAlchemy-JSONField==0.8.0 +sshpubkeys==3.1.0 +sshtunnel==0.1.5 +tabulate==0.8.7 +tenacity==4.12.0 +termcolor==1.1.0 +testpath==0.4.4 +text-unidecode==1.2 +textwrap3==0.9.2 +thrift==0.13.0 +thrift-sasl==0.4.1 +tokenize-rt==3.2.0 +toml==0.10.0 +tornado==5.1.1 +tqdm==4.43.0 +traceback2==1.4.0 +traitlets==4.3.3 +typing==3.7.4.1 +typing-extensions==3.7.4.1 +tzlocal==1.5.1 +unicodecsv==0.14.1 +Unidecode==1.1.1 +unittest2==1.1.0 +uritemplate==3.0.1 +urllib3==1.25.8 +vertica-python==0.10.2 +vine==1.3.0 +virtualenv==20.0.15 +wcwidth==0.1.9 +webencodings==0.5.1 +websocket-client==0.57.0 +Werkzeug==0.16.1 +wrapt==1.12.1 +WTForms==2.2.1 +xmltodict==0.12.0 +yamllint==1.21.0 +zdesk==2.7.1 +zipp==1.2.0 +zope.deprecation==4.4.0 diff --git a/requirements/requirements-python3.5.txt b/requirements/requirements-python3.5.txt new file mode 100644 index 00000000000000..c4754a9fbfd06d --- /dev/null +++ b/requirements/requirements-python3.5.txt @@ -0,0 +1,315 @@ +adal==1.2.2 +alabaster==0.7.12 +alembic==1.4.2 +amqp==2.5.2 +analytics-python==1.2.9 +ansiwrap==0.8.4 +# Editable install with no version control (apache-airflow==1.10.9) +apispec==3.3.0 +appdirs==1.4.3 +argcomplete==1.11.1 +asn1crypto==1.3.0 +aspy.yaml==1.3.0 +astroid==2.3.3 +atlasclient==1.0.0 +attrs==19.3.0 +aws-sam-translator==1.22.0 +aws-xray-sdk==2.4.3 +azure-common==1.1.25 +azure-cosmos==3.1.2 +azure-datalake-store==0.0.48 +azure-mgmt-containerinstance==1.5.0 +azure-mgmt-resource==8.0.1 +azure-nspkg==3.0.2 +azure-storage==0.36.0 +azure-storage-blob==2.1.0 +azure-storage-common==2.1.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +beautifulsoup4==4.7.1 +billiard==3.6.3.0 +blinker==1.4 +boto==2.49.0 +boto3==1.12.31 +botocore==1.15.31 +cached-property==1.5.1 +cachetools==4.0.0 +cassandra-driver==3.20.2 +cattrs==0.9.0 +celery==4.4.2 +certifi==2019.11.28 +cffi==1.14.0 +cfgv==2.0.1 +cfn-lint==0.29.2 +cgroupspy==0.1.6 +chardet==3.0.4 +click==6.7 +cloudant==0.5.10 +colorama==0.4.3 +colorlog==4.0.2 +configparser==3.5.3 +coverage==5.0.4 +croniter==0.3.31 +cryptography==2.8 +cx-Oracle==7.3.0 +datadog==0.35.0 +decorator==4.4.2 +defusedxml==0.6.0 +dill==0.3.1.1 +distlib==0.3.0 +dnspython==1.16.0 +docker==3.7.3 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.16 +ecdsa==0.15 +elasticsearch==5.5.3 +elasticsearch-dsl==5.4.0 +entrypoints==0.3 +fastavro==0.23.0 +filelock==3.0.12 +flake8==3.7.9 +flake8-colors==0.1.6 +flaky==3.6.1 +Flask==1.1.1 +Flask-Admin==1.5.4 +Flask-AppBuilder==1.13.1 +Flask-Babel==0.12.2 +Flask-Bcrypt==0.7.1 +Flask-Caching==1.3.3 +Flask-JWT-Extended==3.24.1 +Flask-Login==0.4.1 +Flask-OpenID==1.2.5 +Flask-SQLAlchemy==2.4.1 +flask-swagger==0.2.13 +Flask-WTF==0.14.3 +flower==0.9.3 +freezegun==0.3.15 +fsspec==0.7.0 +funcsigs==1.0.2 +future==0.18.2 +future-fstrings==1.2.0 +gcsfs==0.6.1 +google-api-core==1.16.0 +google-api-python-client==1.8.0 +google-auth==1.12.0 +google-auth-httplib2==0.0.3 +google-auth-oauthlib==0.4.1 +google-cloud-bigquery==1.24.0 +google-cloud-bigtable==1.2.1 +google-cloud-container==0.4.0 +google-cloud-core==1.3.0 +google-cloud-dlp==0.13.0 +google-cloud-language==1.3.0 +google-cloud-secret-manager==0.2.0 +google-cloud-spanner==1.15.0 +google-cloud-speech==1.3.2 +google-cloud-storage==1.26.0 +google-cloud-texttospeech==1.0.1 +google-cloud-translate==2.0.1 +google-cloud-videointelligence==1.14.0 +google-cloud-vision==1.0.0 +google-resumable-media==0.5.0 +googleapis-common-protos==1.51.0 +graphviz==0.13.2 +grpc-google-iam-v1==0.12.3 +grpcio==1.27.2 +grpcio-gcp==0.2.2 +gunicorn==19.10.0 +hdfs==2.5.8 +hmsclient==0.1.1 +httplib2==0.17.0 +hvac==0.10.0 +identify==1.4.13 +idna==2.9 +ijson==2.6.1 +imagesize==1.2.0 +importlib-metadata==1.6.0 +importlib-resources==1.4.0 +inflection==0.3.1 +ipdb==0.13.2 +ipython==7.9.0 +ipython-genutils==0.2.0 +iso8601==0.1.12 +isodate==0.6.0 +itsdangerous==1.1.0 +JayDeBeApi==1.1.1 +jedi==0.16.0 +Jinja2==2.10.3 +jira==2.0.0 +jmespath==0.9.5 +JPype1==0.7.1 +json-merge-patch==0.2 +jsondiff==1.1.2 +jsonpatch==1.25 +jsonpickle==1.3 +jsonpointer==2.0 +jsonschema==3.2.0 +jupyter-client==6.1.2 +jupyter-core==4.6.3 +kombu==4.6.8 +kubernetes==11.0.0 +lazy-object-proxy==1.4.3 +ldap3==2.7 +lockfile==0.12.2 +Mako==1.1.2 +Markdown==2.6.11 +MarkupSafe==1.1.1 +marshmallow==2.19.5 +marshmallow-enum==1.5.1 +marshmallow-sqlalchemy==0.18.0 +mccabe==0.6.1 +mock==3.0.5 +mongomock==3.19.0 +more-itertools==8.2.0 +moto==1.3.14 +msrest==0.6.11 +msrestazure==0.6.3 +multi-key-dict==2.0.3 +mypy==0.720 +mypy-extensions==0.4.3 +mysqlclient==1.3.14 +nbclient==0.1.0 +nbformat==5.0.4 +networkx==2.4 +nodeenv==1.3.5 +nteract-scrapbook==0.3.1 +ntlm-auth==1.4.0 +numpy==1.18.2 +oauthlib==3.1.0 +oscrypto==1.2.0 +packaging==20.3 +pandas==0.25.3 +pandas-gbq==0.13.1 +papermill==2.0.0 +parameterized==0.7.1 +paramiko==2.7.1 +parso==0.6.2 +pathlib2==2.3.5 +pathspec==0.7.0 +pbr==5.4.4 +pendulum==1.4.4 +pexpect==4.8.0 +pickleshare==0.7.5 +pinotdb==0.1.1 +pluggy==0.13.1 +pre-commit==1.21.0 +presto-python-client==0.7.0 +prison==0.1.0 +prompt-toolkit==2.0.10 +protobuf==3.11.3 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +pyarrow==0.16.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycodestyle==2.5.0 +pycparser==2.20 +pycryptodomex==3.9.7 +pydata-google-auth==0.3.0 +pydruid==0.5.8 +pyflakes==2.1.1 +Pygments==2.6.1 +PyHive==0.6.2 +PyJWT==1.7.1 +pykerberos==1.2.1 +pymongo==3.10.1 +pymssql==2.1.4 +PyNaCl==1.3.0 +pyOpenSSL==19.1.0 +pyparsing==2.4.6 +pyrsistent==0.16.0 +pysftp==0.2.9 +PySmbClient==0.1.5 +pytest==5.4.1 +pytest-cov==2.8.1 +pytest-instafail==0.4.1.post0 +python-daemon==2.1.2 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-http-client==3.2.6 +python-jenkins==1.7.0 +python-jose==3.1.0 +python3-openid==3.1.0 +pytz==2019.3 +pytzdata==2019.3 +pywinrm==0.4.1 +PyYAML==5.3.1 +pyzmq==19.0.0 +qds-sdk==1.15.0 +redis==3.4.1 +requests==2.23.0 +requests-futures==0.9.4 +requests-kerberos==0.12.0 +requests-mock==1.7.0 +requests-ntlm==1.1.0 +requests-oauthlib==1.3.0 +requests-toolbelt==0.9.1 +responses==0.10.12 +rsa==4.0 +s3transfer==0.3.3 +sasl==0.2.1 +sendgrid==5.6.0 +sentinels==1.0.0 +sentry-sdk==0.14.3 +setproctitle==1.1.10 +six==1.14.0 +slackclient==1.3.2 +snowballstemmer==2.0.0 +snowflake-connector-python==2.2.2 +snowflake-sqlalchemy==1.2.2 +soupsieve==2.0 +Sphinx==2.4.4 +sphinx-argparse==0.2.5 +sphinx-autoapi==1.0.0 +sphinx-jinja==1.1.1 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-dotnetdomain==0.4 +sphinxcontrib-golangdomain==0.2.0.dev0 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-httpdomain==1.7.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.15 +SQLAlchemy-JSONField==0.9.0 +sshpubkeys==3.1.0 +sshtunnel==0.1.5 +tabulate==0.8.7 +tenacity==4.12.0 +termcolor==1.1.0 +text-unidecode==1.2 +textwrap3==0.9.2 +thrift==0.13.0 +thrift-sasl==0.4.1 +tokenize-rt==3.2.0 +toml==0.10.0 +tornado==5.1.1 +tqdm==4.43.0 +traitlets==4.3.3 +typed-ast==1.4.1 +typing==3.7.4.1 +typing-extensions==3.7.4.1 +tzlocal==1.5.1 +unicodecsv==0.14.1 +Unidecode==1.1.1 +uritemplate==3.0.1 +urllib3==1.25.8 +vertica-python==0.10.2 +vine==1.3.0 +virtualenv==20.0.15 +wcwidth==0.1.9 +websocket-client==0.57.0 +Werkzeug==0.16.1 +wrapt==1.12.1 +WTForms==2.2.1 +xmltodict==0.12.0 +yamllint==1.21.0 +zdesk==2.7.1 +zipp==1.2.0 +zope.deprecation==4.4.0 diff --git a/requirements.txt b/requirements/requirements-python3.6.txt similarity index 97% rename from requirements.txt rename to requirements/requirements-python3.6.txt index 3d7c9431abf9a6..c4aac01f30e4ae 100644 --- a/requirements.txt +++ b/requirements/requirements-python3.6.txt @@ -5,7 +5,7 @@ amqp==2.5.2 analytics-python==1.2.9 ansiwrap==0.8.4 # Editable install with no version control (apache-airflow==1.10.9) -apispec==3.3.0 +apispec==1.3.3 appdirs==1.4.3 argcomplete==1.11.1 asn1crypto==1.3.0 @@ -29,6 +29,7 @@ backcall==0.1.0 bcrypt==3.1.7 beautifulsoup4==4.7.1 billiard==3.6.3.0 +black==19.10b0 blinker==1.4 boto==2.49.0 boto3==1.12.26 @@ -74,8 +75,8 @@ flake8-colors==0.1.6 flaky==3.6.1 Flask==1.1.1 Flask-Admin==1.5.4 -Flask-AppBuilder==1.13.1 -Flask-Babel==0.12.2 +Flask-AppBuilder==2.3.0 +Flask-Babel==1.0.0 Flask-Bcrypt==0.7.1 Flask-Caching==1.3.3 Flask-JWT-Extended==3.24.1 @@ -186,7 +187,6 @@ papermill==2.0.0 parameterized==0.7.1 paramiko==2.7.1 parso==0.6.2 -pathlib2==2.3.5 pathspec==0.7.0 pbr==5.4.4 pendulum==1.4.4 @@ -196,7 +196,7 @@ pinotdb==0.1.1 pluggy==0.13.1 pre-commit==1.21.0 presto-python-client==0.7.0 -prison==0.1.0 +prison==0.1.3 prompt-toolkit==2.0.10 protobuf==3.11.3 psutil==5.7.0 @@ -241,6 +241,7 @@ PyYAML==5.3.1 pyzmq==19.0.0 qds-sdk==1.15.0 redis==3.4.1 +regex==2020.2.20 requests==2.23.0 requests-futures==0.9.4 requests-kerberos==0.12.0 @@ -278,6 +279,7 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.4 SQLAlchemy==1.3.15 SQLAlchemy-JSONField==0.9.0 +SQLAlchemy-Utils==0.36.3 sshpubkeys==3.1.0 sshtunnel==0.1.5 tabulate==0.8.6 @@ -287,7 +289,6 @@ text-unidecode==1.2 textwrap3==0.9.2 thrift==0.13.0 thrift-sasl==0.4.1 -tokenize-rt==3.2.0 toml==0.10.0 tornado==5.1.1 tqdm==4.43.0 diff --git a/requirements/requirements-python3.7.txt b/requirements/requirements-python3.7.txt new file mode 100644 index 00000000000000..fa3eb883061600 --- /dev/null +++ b/requirements/requirements-python3.7.txt @@ -0,0 +1,313 @@ +adal==1.2.2 +alabaster==0.7.12 +alembic==1.4.2 +amqp==2.5.2 +analytics-python==1.2.9 +ansiwrap==0.8.4 +# Editable install with no version control (apache-airflow==1.10.9) +apispec==1.3.3 +appdirs==1.4.3 +argcomplete==1.11.1 +asn1crypto==1.3.0 +astroid==2.3.3 +atlasclient==1.0.0 +attrs==19.3.0 +aws-sam-translator==1.22.0 +aws-xray-sdk==2.4.3 +azure-common==1.1.25 +azure-cosmos==3.1.2 +azure-datalake-store==0.0.48 +azure-mgmt-containerinstance==1.5.0 +azure-mgmt-resource==8.0.1 +azure-nspkg==3.0.2 +azure-storage==0.36.0 +azure-storage-blob==2.1.0 +azure-storage-common==2.1.0 +Babel==2.8.0 +backcall==0.1.0 +bcrypt==3.1.7 +beautifulsoup4==4.7.1 +billiard==3.6.3.0 +black==19.10b0 +blinker==1.4 +boto==2.49.0 +boto3==1.12.26 +botocore==1.15.26 +cached-property==1.5.1 +cachetools==4.0.0 +cassandra-driver==3.20.2 +cattrs==0.9.0 +celery==4.4.2 +certifi==2019.11.28 +cffi==1.14.0 +cfgv==3.1.0 +cfn-lint==0.29.0 +cgroupspy==0.1.6 +chardet==3.0.4 +click==6.7 +cloudant==0.5.10 +colorama==0.4.3 +colorlog==4.0.2 +configparser==3.5.3 +coverage==5.0.4 +croniter==0.3.31 +cryptography==2.8 +cx-Oracle==7.3.0 +datadog==0.35.0 +decorator==4.4.2 +defusedxml==0.6.0 +dill==0.3.1.1 +distlib==0.3.0 +dnspython==1.16.0 +docker==3.7.3 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.16 +ecdsa==0.15 +elasticsearch==5.5.3 +elasticsearch-dsl==5.4.0 +entrypoints==0.3 +fastavro==0.22.13 +filelock==3.0.12 +flake8==3.7.9 +flake8-colors==0.1.6 +flaky==3.6.1 +Flask==1.1.1 +Flask-Admin==1.5.4 +Flask-AppBuilder==2.3.0 +Flask-Babel==1.0.0 +Flask-Bcrypt==0.7.1 +Flask-Caching==1.3.3 +Flask-JWT-Extended==3.24.1 +Flask-Login==0.4.1 +Flask-OpenID==1.2.5 +Flask-SQLAlchemy==2.4.1 +flask-swagger==0.2.13 +Flask-WTF==0.14.3 +flower==0.9.3 +freezegun==0.3.15 +fsspec==0.6.3 +funcsigs==1.0.2 +future==0.18.2 +future-fstrings==1.2.0 +gcsfs==0.6.0 +google-api-core==1.16.0 +google-api-python-client==1.8.0 +google-auth==1.11.3 +google-auth-httplib2==0.0.3 +google-auth-oauthlib==0.4.1 +google-cloud-bigquery==1.24.0 +google-cloud-bigtable==1.2.1 +google-cloud-container==0.4.0 +google-cloud-core==1.3.0 +google-cloud-dlp==0.13.0 +google-cloud-language==1.3.0 +google-cloud-secret-manager==0.2.0 +google-cloud-spanner==1.15.0 +google-cloud-speech==1.3.2 +google-cloud-storage==1.26.0 +google-cloud-texttospeech==1.0.1 +google-cloud-translate==2.0.1 +google-cloud-videointelligence==1.14.0 +google-cloud-vision==1.0.0 +google-resumable-media==0.5.0 +googleapis-common-protos==1.51.0 +graphviz==0.13.2 +grpc-google-iam-v1==0.12.3 +grpcio==1.27.2 +grpcio-gcp==0.2.2 +gunicorn==19.10.0 +hdfs==2.5.8 +hmsclient==0.1.1 +httplib2==0.17.0 +hvac==0.10.0 +identify==1.4.11 +idna==2.9 +ijson==2.6.1 +imagesize==1.2.0 +importlib-metadata==1.5.0 +inflection==0.3.1 +ipdb==0.13.2 +ipython==7.13.0 +ipython-genutils==0.2.0 +iso8601==0.1.12 +isodate==0.6.0 +itsdangerous==1.1.0 +JayDeBeApi==1.1.1 +jedi==0.16.0 +Jinja2==2.10.3 +jira==2.0.0 +jmespath==0.9.5 +JPype1==0.7.1 +json-merge-patch==0.2 +jsondiff==1.1.2 +jsonpatch==1.25 +jsonpickle==1.3 +jsonpointer==2.0 +jsonschema==3.2.0 +jupyter-client==6.1.0 +jupyter-core==4.6.3 +kombu==4.6.8 +kubernetes==11.0.0 +lazy-object-proxy==1.4.3 +ldap3==2.7 +lockfile==0.12.2 +Mako==1.1.2 +Markdown==2.6.11 +MarkupSafe==1.1.1 +marshmallow==3.5.1 +marshmallow-enum==1.5.1 +marshmallow-sqlalchemy==0.22.3 +mccabe==0.6.1 +mock==4.0.2 +mongomock==3.19.0 +more-itertools==8.2.0 +moto==1.3.14 +msrest==0.6.11 +msrestazure==0.6.3 +multi-key-dict==2.0.3 +mypy==0.720 +mypy-extensions==0.4.3 +mysqlclient==1.3.14 +nbclient==0.1.0 +nbformat==5.0.4 +networkx==2.4 +nodeenv==1.3.5 +nteract-scrapbook==0.3.1 +ntlm-auth==1.4.0 +numpy==1.18.2 +oauthlib==3.1.0 +oscrypto==1.2.0 +packaging==20.3 +pandas==0.25.3 +pandas-gbq==0.13.1 +papermill==2.0.0 +parameterized==0.7.1 +paramiko==2.7.1 +parso==0.6.2 +pathspec==0.7.0 +pbr==5.4.4 +pendulum==1.4.4 +pexpect==4.8.0 +pickleshare==0.7.5 +pinotdb==0.1.1 +pluggy==0.13.1 +pre-commit==2.2.0 +presto-python-client==0.7.0 +prison==0.1.2 +prompt-toolkit==3.0.4 +protobuf==3.11.3 +psutil==5.7.0 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +pyarrow==0.16.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycodestyle==2.5.0 +pycparser==2.20 +pycryptodomex==3.9.7 +pydata-google-auth==0.3.0 +pydruid==0.5.8 +pyflakes==2.1.1 +Pygments==2.6.1 +PyHive==0.6.2 +PyJWT==1.7.1 +pykerberos==1.2.1 +pymongo==3.10.1 +pymssql==2.1.4 +PyNaCl==1.3.0 +pyOpenSSL==19.1.0 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pysftp==0.2.9 +PySmbClient==0.1.5 +pytest==5.4.1 +pytest-cov==2.8.1 +pytest-instafail==0.4.1.post0 +python-daemon==2.1.2 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-http-client==3.2.6 +python-jenkins==1.7.0 +python-jose==3.1.0 +python3-openid==3.1.0 +pytz==2019.3 +pytzdata==2019.3 +pywinrm==0.4.1 +PyYAML==5.3.1 +pyzmq==19.0.0 +qds-sdk==1.15.0 +redis==3.4.1 +regex==2020.2.20 +requests==2.23.0 +requests-futures==0.9.4 +requests-kerberos==0.12.0 +requests-mock==1.7.0 +requests-ntlm==1.1.0 +requests-oauthlib==1.3.0 +requests-toolbelt==0.9.1 +responses==0.10.12 +rsa==4.0 +s3transfer==0.3.3 +sasl==0.2.1 +sendgrid==5.6.0 +sentinels==1.0.0 +sentry-sdk==0.14.3 +setproctitle==1.1.10 +six==1.14.0 +slackclient==1.3.2 +snowballstemmer==2.0.0 +snowflake-connector-python==2.2.2 +snowflake-sqlalchemy==1.2.2 +soupsieve==2.0 +Sphinx==2.4.4 +sphinx-argparse==0.2.5 +sphinx-autoapi==1.0.0 +sphinx-jinja==1.1.1 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-dotnetdomain==0.4 +sphinxcontrib-golangdomain==0.2.0.dev0 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-httpdomain==1.7.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +SQLAlchemy==1.3.15 +SQLAlchemy-JSONField==0.9.0 +SQLAlchemy-Utils==0.36.3 +sshpubkeys==3.1.0 +sshtunnel==0.1.5 +tabulate==0.8.6 +tenacity==4.12.0 +termcolor==1.1.0 +text-unidecode==1.2 +textwrap3==0.9.2 +thrift==0.13.0 +thrift-sasl==0.4.1 +toml==0.10.0 +tornado==5.1.1 +tqdm==4.43.0 +traitlets==4.3.3 +typed-ast==1.4.1 +typing-extensions==3.7.4.1 +tzlocal==1.5.1 +unicodecsv==0.14.1 +Unidecode==1.1.1 +uritemplate==3.0.1 +urllib3==1.25.8 +vertica-python==0.10.2 +vine==1.3.0 +virtualenv==20.0.13 +wcwidth==0.1.8 +websocket-client==0.57.0 +Werkzeug==0.16.1 +wrapt==1.12.1 +WTForms==2.2.1 +xmltodict==0.12.0 +yamllint==1.20.0 +zdesk==2.7.1 +zipp==3.1.0 +zope.deprecation==4.4.0 diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index caae24db29c6b6..e98feacdabcf45 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -40,7 +40,7 @@ function verbose_docker { # Common environment that is initialized by both Breeze and CI scripts function initialize_common_environment { # default python version - PYTHON_VERSION=${PYTHON_VERSION:="3.5"} + PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.5"} AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${MY_DIR}/../../" && pwd)} export AIRFLOW_SOURCES @@ -137,8 +137,9 @@ function initialize_common_environment { # We use pulled docker image cache by default to speed up the builds export DOCKER_CACHE=${DOCKER_CACHE:="pulled"} - # We use requirements.txt by default rather than trying to upgrade to latest versions - export UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD=${UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD:="false"} + # By default we are not upgrading to latest requirements + # This will only be done in cron jobs and when run with breeze + export UPGRADE_TO_LATEST_REQUIREMENTS=${UPGRADE_TO_LATEST_REQUIREMENTS:="false"} STAT_BIN=stat if [[ "${OSTYPE}" == "darwin"* ]]; then @@ -149,7 +150,7 @@ function initialize_common_environment { export AIRFLOW_VERSION # default version for dockerhub images - export PYTHON_VERSION_FOR_DEFAULT_DOCKERHUB_IMAGE=3.6 + export DEFAULT_PYTHON_MAJOR_MINOR_VERSION=3.6 if [[ ${CI:="false"} == "true" ]]; then @@ -198,7 +199,7 @@ dist / hooks /opt/airflow/ logs /root/airflow/ pytest.ini /opt/airflow/ -requirements.txt /opt/airflow/ +requirements /opt/airflow/ scripts /opt/airflow/ scripts/ci/in_container/entrypoint_ci.sh / setup.cfg /opt/airflow/ @@ -267,7 +268,7 @@ function remove_cache_directory() { function check_file_md5sum { local FILE="${1}" local MD5SUM - local MD5SUM_CACHE_DIR="${BUILD_CACHE_DIR}/${DEFAULT_BRANCH}/${PYTHON_VERSION}/${THE_IMAGE_TYPE}" + local MD5SUM_CACHE_DIR="${BUILD_CACHE_DIR}/${DEFAULT_BRANCH}/${PYTHON_MAJOR_MINOR_VERSION}/${THE_IMAGE_TYPE}" mkdir -pv "${MD5SUM_CACHE_DIR}" MD5SUM=$(md5sum "${FILE}") local MD5SUM_FILE @@ -297,7 +298,7 @@ function check_file_md5sum { function move_file_md5sum { local FILE="${1}" local MD5SUM_FILE - local MD5SUM_CACHE_DIR="${BUILD_CACHE_DIR}/${DEFAULT_BRANCH}/${PYTHON_VERSION}/${THE_IMAGE_TYPE}" + local MD5SUM_CACHE_DIR="${BUILD_CACHE_DIR}/${DEFAULT_BRANCH}/${PYTHON_MAJOR_MINOR_VERSION}/${THE_IMAGE_TYPE}" mkdir -pv "${MD5SUM_CACHE_DIR}" MD5SUM_FILE="${MD5SUM_CACHE_DIR}"/$(basename "${FILE}").md5sum local MD5SUM_FILE_NEW @@ -514,13 +515,13 @@ function confirm_image_rebuild() { esac elif [[ -t 0 ]]; then # Check if this script is run interactively with stdin open and terminal attached - "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE} (might take some time)" + "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" RES=$? elif [[ ${DETECTED_TERMINAL:=$(tty)} != "not a tty" ]]; then # Make sure to use output of tty rather than stdin/stdout when available - this way confirm # will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks) # shellcheck disable=SC2094 - "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE} (might take some time)" \ + "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \ <"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}" RES=$? export DETECTED_TERMINAL @@ -529,7 +530,7 @@ function confirm_image_rebuild() { # Make sure to use /dev/tty first rather than stdin/stdout when available - this way confirm # will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks) # shellcheck disable=SC2094 - "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE} (might take some time)" \ + "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \ <"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}" RES=$? else @@ -542,7 +543,7 @@ function confirm_image_rebuild() { set -e if [[ ${RES} == "1" ]]; then print_info - print_info "Skipping rebuilding the image ${THE_IMAGE_TYPE}" + print_info "Skipping rebuilding the image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" print_info SKIP_REBUILD="true" # Force "no" also to subsequent questions so that if you answer it once, you are not asked @@ -554,7 +555,7 @@ function confirm_image_rebuild() { echo >&2 "ERROR: The ${THE_IMAGE_TYPE} needs to be rebuilt - it is outdated. " echo >&2 " Make sure you build the images bu running" echo >&2 - echo >&2 " ./breeze --python ${PYTHON_VERSION}" build-only + echo >&2 " ./breeze --python ${PYTHON_MAJOR_MINOR_VERSION}" build-only echo >&2 echo >&2 " If you run it via pre-commit as individual hook, you can run 'pre-commit run build'." echo >&2 @@ -579,7 +580,7 @@ function set_current_image_variables { export AIRFLOW_REMOTE_MANIFEST_IMAGE="" fi - if [[ "${PYTHON_VERSION_FOR_DEFAULT_DOCKERHUB_IMAGE}" == "${PYTHON_VERSION}" ]]; then + if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" == "${PYTHON_MAJOR_MINOR_VERSION}" ]]; then export DEFAULT_IMAGE="${AIRFLOW_IMAGE_DEFAULT}" else export DEFAULT_IMAGE="" @@ -696,6 +697,7 @@ function compare_layers() { echo echo echo "WARNING! Your image and the dockerhub image differ signifcantly" + echo "WARNING! Your image and the dockerhub image differ signifcantly" echo echo "Forcing pulling the images. It will be faster than rebuilding usually." echo "You can avoid it by setting SKIP_CHECK_REMOTE_IMAGE to true" @@ -1081,7 +1083,7 @@ Docker pulling ${IMAGE}. function print_build_info() { print_info - print_info "Airflow ${AIRFLOW_VERSION} Python: ${PYTHON_VERSION}. Image description: ${IMAGE_DESCRIPTION}" + print_info "Airflow ${AIRFLOW_VERSION} Python: ${PYTHON_MAJOR_MINOR_VERSION}. Image description: ${IMAGE_DESCRIPTION}" print_info } @@ -1156,11 +1158,12 @@ Docker building ${AIRFLOW_CI_IMAGE}. set +u verbose_docker build \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ + --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${BRANCH_NAME}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ --build-arg AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" \ - --build-arg UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD="${UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD}" \ + --build-arg UPGRADE_TO_LATEST_REQUIREMENTS="${UPGRADE_TO_LATEST_REQUIREMENTS}" \ "${DOCKER_CACHE_CI_DIRECTIVE[@]}" \ -t "${AIRFLOW_CI_IMAGE}" \ --target "${TARGET_IMAGE}" \ @@ -1185,7 +1188,7 @@ function remove_all_images() { verbose_docker rmi "${AIRFLOW_CI_IMAGE}" || true echo echo "###################################################################" - echo "NOTE!! Removed Airflow images for Python version ${PYTHON_VERSION}." + echo "NOTE!! Removed Airflow images for Python version ${PYTHON_MAJOR_MINOR_VERSION}." echo " But the disk space in docker will be reclaimed only after" echo " running 'docker system prune' command." echo "###################################################################" @@ -1234,13 +1237,13 @@ function fix_group_permissions() { } function set_common_image_variables { - export AIRFLOW_CI_BASE_TAG="${DEFAULT_BRANCH}-python${PYTHON_VERSION}-ci" + export AIRFLOW_CI_BASE_TAG="${DEFAULT_BRANCH}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest" export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest" export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}" export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${DEFAULT_BRANCH}-ci" - export PYTHON_BASE_IMAGE="python:${PYTHON_VERSION}-slim-buster" - export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_VERSION}" + export PYTHON_BASE_IMAGE="python:${PYTHON_MAJOR_MINOR_VERSION}-slim-buster" + export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}" } @@ -1272,6 +1275,21 @@ function push_image() { fi } +function run_generate_requirements() { + docker run "${EXTRA_DOCKER_FLAGS[@]}" \ + --entrypoint "/usr/local/bin/dumb-init" \ + --env PYTHONDONTWRITEBYTECODE \ + --env VERBOSE \ + --env VERBOSE_COMMANDS \ + --env HOST_USER_ID="$(id -ur)" \ + --env HOST_GROUP_ID="$(id -gr)" \ + --env UPGRADE_WHILE_GENERATING_REQUIREMENTS \ + --env PYTHON_MAJOR_MINOR_VERSION \ + --rm \ + "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_generate_requirements.sh" \ + | tee -a "${OUTPUT_LOG}" +} function set_mysql_encoding() { export MYSQL_ENCODING="utf8mb4" diff --git a/scripts/ci/ci_before_install.sh b/scripts/ci/ci_before_install.sh index d88ffb45bf1eaa..6c5d6a63700043 100755 --- a/scripts/ci/ci_before_install.sh +++ b/scripts/ci/ci_before_install.sh @@ -20,7 +20,7 @@ set -x # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" -export UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD="false" +export UPGRADE_TO_LATEST_REQUIREMENTS="false" # In case of CRON jobs on Travis we run builds without cache if [[ "${TRAVIS_EVENT_TYPE:=}" == "cron" ]]; then @@ -29,7 +29,7 @@ if [[ "${TRAVIS_EVENT_TYPE:=}" == "cron" ]]; then echo export DOCKER_CACHE="no-cache" export PULL_BASE_IMAGES="true" - export UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD="true" + export UPGRADE_TO_LATEST_REQUIREMENTS="true" fi diff --git a/scripts/ci/ci_build_dockerhub.sh b/scripts/ci/ci_build_dockerhub.sh index e068f574c1d096..7bf149de15b87e 100755 --- a/scripts/ci/ci_build_dockerhub.sh +++ b/scripts/ci/ci_build_dockerhub.sh @@ -51,9 +51,9 @@ else echo "DOCKER_TAG=${DOCKER_TAG}" fi -[[ ${DOCKER_TAG:=} =~ ${DEFAULT_BRANCH}-python([0-9.]*)(.*) ]] && export PYTHON_VERSION=${BASH_REMATCH[1]} +[[ ${DOCKER_TAG:=} =~ ${DEFAULT_BRANCH}-python([0-9.]*)(.*) ]] && export PYTHON_MAJOR_MINOR_VERSION=${BASH_REMATCH[1]} -if [[ -z ${PYTHON_VERSION:=} ]]; then +if [[ -z ${PYTHON_MAJOR_MINOR_VERSION:=} ]]; then echo echo "Error! Wrong DOCKER_TAG" echo "The tag '${DOCKER_TAG}' should follow the pattern ${DEFAULT_BRANCH}-pythonX.Y[-ci]" @@ -61,7 +61,7 @@ if [[ -z ${PYTHON_VERSION:=} ]]; then exit 1 fi -echo "Detected PYTHON_VERSION=${PYTHON_VERSION}" +echo "Detected PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}" echo # shellcheck source=scripts/ci/_script_init.sh diff --git a/scripts/ci/ci_check_license.sh b/scripts/ci/ci_check_license.sh index ea896a13331a44..711f9c49f9863e 100755 --- a/scripts/ci/ci_check_license.sh +++ b/scripts/ci/ci_check_license.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. export AIRFLOW_MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="true" -export PYTHON_VERSION=${PYTHON_VERSION:-3.5} +export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.5} # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/ci_docs.sh index f9c1fc6668e29c..4373bb19c823bb 100755 --- a/scripts/ci/ci_docs.sh +++ b/scripts/ci/ci_docs.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -export PYTHON_VERSION=${PYTHON_VERSION:-3.5} +export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.5} # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" diff --git a/scripts/ci/ci_fix_ownership.sh b/scripts/ci/ci_fix_ownership.sh index 2c0515ecb4c877..37f15afc704633 100755 --- a/scripts/ci/ci_fix_ownership.sh +++ b/scripts/ci/ci_fix_ownership.sh @@ -20,13 +20,13 @@ # Fixes ownership for files created inside container (files owned by root will be owned by host user) # -export PYTHON_VERSION=${PYTHON_VERSION:-3.5} +export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.5} # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" export AIRFLOW_CI_IMAGE=\ -${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_VERSION}-ci +${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci docker-compose \ -f "${MY_DIR}/docker-compose/base.yml" \ diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/ci_flake8.sh index 0b49dbc46a750e..aa4c566dd57f7c 100755 --- a/scripts/ci/ci_flake8.sh +++ b/scripts/ci/ci_flake8.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -export PYTHON_VERSION=${PYTHON_VERSION:-3.5} +export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.5} # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" diff --git a/scripts/ci/ci_generate_requirements.sh b/scripts/ci/ci_generate_requirements.sh index 10e7be1497a392..f7d7d9cea6f186 100755 --- a/scripts/ci/ci_generate_requirements.sh +++ b/scripts/ci/ci_generate_requirements.sh @@ -15,26 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -export PYTHON_VERSION=${PYTHON_VERSION:-3.5} +export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.5} # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" -function run_generate_requirements() { - docker run "${EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint "/usr/local/bin/dumb-init" \ - --env PYTHONDONTWRITEBYTECODE \ - --env VERBOSE \ - --env VERBOSE_COMMANDS \ - --env HOST_USER_ID="$(id -ur)" \ - --env HOST_GROUP_ID="$(id -gr)" \ - --env UPGRADE_WHILE_GENERATING_REQUIREMENTS \ - --rm \ - "${AIRFLOW_CI_IMAGE}" \ - "--" "/opt/airflow/scripts/ci/in_container/run_generate_requirements.sh" \ - | tee -a "${OUTPUT_LOG}" -} - prepare_build rebuild_ci_image_if_needed diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/ci_mypy.sh index 99cbd3aa33ff4f..7c3b975162a83b 100755 --- a/scripts/ci/ci_mypy.sh +++ b/scripts/ci/ci_mypy.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -export PYTHON_VERSION=${PYTHON_VERSION:-3.5} +export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.5} # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" diff --git a/scripts/ci/ci_run_all_static_tests.sh b/scripts/ci/ci_run_all_static_tests.sh index f2e72b920fb4b1..9cbfae2fcd0309 100755 --- a/scripts/ci/ci_run_all_static_tests.sh +++ b/scripts/ci/ci_run_all_static_tests.sh @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -export PYTHON_VERSION=${PYTHON_VERSION:-3.5} +export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.5} # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 7ec5864532cd16..91cc94208507a6 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -59,6 +59,7 @@ services: - DB_RESET - ENABLED_SYSTEMS - RUN_SYSTEM_TESTS + - PYTHON_MAJOR_MINOR_VERSION volumes: # Pass docker to inside of the container so that Kind and Moto tests can use it. - /var/run/docker.sock:/var/run/docker.sock diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 95283b4ff0be12..94c934fd2892db 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -49,7 +49,7 @@ services: - ../../../hooks:/opt/airflow/hooks:cached - ../../../logs:/root/airflow/logs:cached - ../../../pytest.ini:/opt/airflow/pytest.ini:cached - - ../../../requirements.txt:/opt/airflow/requirements.txt:cached + - ../../../requirements:/opt/airflow/requirements:cached - ../../../scripts:/opt/airflow/scripts:cached - ../../../scripts/ci/in_container/entrypoint_ci.sh:/entrypoint_ci.sh:cached - ../../../setup.cfg:/opt/airflow/setup.cfg:cached diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh index 8f6f5fba945d5b..002419377ea18c 100644 --- a/scripts/ci/in_container/_in_container_utils.sh +++ b/scripts/ci/in_container/_in_container_utils.sh @@ -85,7 +85,8 @@ function in_container_cleanup_pycache() { # function in_container_fix_ownership() { set +o pipefail - sudo find . -user root | sudo xargs chown -v "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference + sudo find . -user root \ + | sudo xargs chown -v "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference >/dev/null 2>&1 set -o pipefail } diff --git a/scripts/ci/in_container/entrypoint_ci.sh b/scripts/ci/in_container/entrypoint_ci.sh index 65cc402620b13a..04efb6bb409a73 100755 --- a/scripts/ci/in_container/entrypoint_ci.sh +++ b/scripts/ci/in_container/entrypoint_ci.sh @@ -26,7 +26,7 @@ TRAVIS=${TRAVIS:=} AIRFLOW_SOURCES=$(cd "${MY_DIR}/../../.." || exit 1; pwd) -PYTHON_VERSION=${PYTHON_VERSION:=3.6} +PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.6} BACKEND=${BACKEND:=sqlite} KUBERNETES_MODE=${KUBERNETES_MODE:=""} KUBERNETES_VERSION=${KUBERNETES_VERSION:=""} @@ -180,7 +180,7 @@ fi if [[ "${ENABLE_KIND_CLUSTER}" == "true" ]]; then - export CLUSTER_NAME="airflow-python-${PYTHON_VERSION}-${KUBERNETES_VERSION}" + export CLUSTER_NAME="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}" "${MY_DIR}/kubernetes/setup_kind_cluster.sh" if [[ ${KIND_CLUSTER_OPERATION} == "stop" ]]; then exit 1 diff --git a/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh b/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh index ffd8420ac67279..74b9bc2c8bc0d3 100755 --- a/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh +++ b/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh @@ -41,11 +41,12 @@ echo start_output_heartbeat "Rebuilding Kubernetes image" 3 docker build \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ + --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH}" \ --build-arg AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" \ - --build-arg UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD="${UPGRADE_TO_LATEST_REQUIREMENTS_IN_DOCKER_BUILD}" \ + --build-arg UPGRADE_TO_LATEST_REQUIREMENTS="${UPGRADE_TO_LATEST_REQUIREMENTS}" \ --build-arg HOME="${HOME}" \ --cache-from "${AIRFLOW_CI_IMAGE}" \ --tag="${AIRFLOW_CI_IMAGE}" \ diff --git a/scripts/ci/in_container/run_docs_build.sh b/scripts/ci/in_container/run_docs_build.sh index 96cbec229be2be..c30c1a048b0420 100755 --- a/scripts/ci/in_container/run_docs_build.sh +++ b/scripts/ci/in_container/run_docs_build.sh @@ -18,8 +18,11 @@ # shellcheck source=scripts/ci/in_container/_in_container_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" +# adding trap to exiting trap +HANDLERS="$( trap -p EXIT | cut -f2 -d \' )" +# shellcheck disable=SC2064 +trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT + sudo rm -rf "$(pwd)/docs/_build/*" sudo rm -rf "$(pwd)/docs/_api/*" sudo -E "$(pwd)/docs/build.sh" - -in_container_fix_ownership diff --git a/scripts/ci/in_container/run_generate_requirements.sh b/scripts/ci/in_container/run_generate_requirements.sh index 9192779a547de9..4c79b5b5025197 100755 --- a/scripts/ci/in_container/run_generate_requirements.sh +++ b/scripts/ci/in_container/run_generate_requirements.sh @@ -18,23 +18,52 @@ # shellcheck source=scripts/ci/in_container/_in_container_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" +# adding trap to exiting trap +HANDLERS="$( trap -p EXIT | cut -f2 -d \' )" +# shellcheck disable=SC2064 +trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT + +# Upgrading requirements will happen only in CRON job to see that we have some +# new requirements released if [[ ${UPGRADE_WHILE_GENERATING_REQUIREMENTS} == "true" ]]; then echo - echo "Upgrading requirements" + echo "Upgrading requirements to latest ones" echo pip install -e ".[${AIRFLOW_EXTRAS}]" --upgrade fi +OLD_REQUIREMENTS_FILE="/tmp/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt" +GENERATED_REQUIREMENTS_FILE="${AIRFLOW_SOURCES}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt" + +echo +echo "Copying requirements ${GENERATED_REQUIREMENTS_FILE} -> ${OLD_REQUIREMENTS_FILE}" +echo +cp "${GENERATED_REQUIREMENTS_FILE}" "${OLD_REQUIREMENTS_FILE}" + echo -echo "Freezing requirements" +echo "Freezing requirements to ${GENERATED_REQUIREMENTS_FILE}" echo pip freeze | \ grep -v "apache_airflow" | \ - grep -v "/opt/airflow" >"${AIRFLOW_SOURCES}/requirements.txt" + grep -v "/opt/airflow" >"${GENERATED_REQUIREMENTS_FILE}" echo -echo "Requirements generated in requirements.txt" +echo "Requirements generated in ${GENERATED_REQUIREMENTS_FILE}" echo -in_container_fix_ownership +set +e +# Fail in case diff shows difference +diff --color=always "${OLD_REQUIREMENTS_FILE}" "${GENERATED_REQUIREMENTS_FILE}" +RES=$? + +if [[ ${RES} != "0" ]]; then + echo + echo " ERROR! Requirements need to be updated!" + echo + echo " Please generate requirements with:" + echo + echo " breeze generate-requirements --python ${PYTHON_MAJOR_MINOR_VERSION}" + echo + exit "${RES}" +fi diff --git a/scripts/ci/pre_commit_ci_build.sh b/scripts/ci/pre_commit_ci_build.sh index 04a2c5de41fae2..28c97b0855534a 100755 --- a/scripts/ci/pre_commit_ci_build.sh +++ b/scripts/ci/pre_commit_ci_build.sh @@ -15,8 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -export REMEMBER_LAST_ANSWER="false" -export PYTHON_VERSION=3.5 +export PYTHON_MAJOR_MINOR_VERSION="${1}" +export REMEMBER_LAST_ANSWER="${2}" # shellcheck source=scripts/ci/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh" diff --git a/scripts/ci/pre_commit_generate_requirements.sh b/scripts/ci/pre_commit_generate_requirements.sh index f61cfc8a67448e..f9e952e8471b1d 100755 --- a/scripts/ci/pre_commit_generate_requirements.sh +++ b/scripts/ci/pre_commit_generate_requirements.sh @@ -18,5 +18,7 @@ export FORCE_ANSWER_TO_QUESTIONS=${FORCE_ANSWER_TO_QUESTIONS:="quit"} export REMEMBER_LAST_ANSWER="true" +export PYTHON_MAJOR_MINOR_VERSION="${1}" + # shellcheck source=scripts/ci/ci_generate_requirements.sh . "$( dirname "${BASH_SOURCE[0]}" )/ci_generate_requirements.sh" diff --git a/setup.py b/setup.py index fddcd03b72fc27..23feafe6e58eb4 100644 --- a/setup.py +++ b/setup.py @@ -593,53 +593,11 @@ def get_dependency_name(dep): return dep.replace(">", '=').replace("<", "=").split("=")[0] -def read_requirements_txt(): - """Returns dictionary of requirements read from requirements.txt""" - with open(os.path.join(my_dir, "requirements.txt"), "rt") as requirement_file: - requirements_content = requirement_file.readlines() - requirements = {} - for requirements_line in requirements_content: - if requirements_line.strip().startswith("#"): - continue - split_requirement = requirements_line.split("==") - requirements[split_requirement[0]] = requirements_line.rstrip("\n") - return requirements - - -def get_extras_required(pinned=False): - """Get dict of extras requirements - pinned if specified.""" - if not pinned: - return EXTRAS_REQUIREMENTS - pinned_extras_required = {} - requirements = read_requirements_txt() - for extra, dependencies in EXTRAS_REQUIREMENTS.items(): - pinned_deps = [] - for dependency in dependencies: - dep_name = get_dependency_name(dependency) - if requirements.get(dep_name): - pinned_deps.append(requirements[dep_name]) - pinned_extras_required[extra] = pinned_deps - return pinned_extras_required - - -def get_install_requires(pinned=False): - """Get list of install requirements - pinned if specified.""" - if not pinned: - return INSTALL_REQUIREMENTS - pinned_install_requires = [] - requirements = read_requirements_txt() - for dependency in INSTALL_REQUIREMENTS: - dep_name = get_dependency_name(dependency) - if requirements.get(dep_name): - pinned_install_requires.append(requirements[dep_name]) - return pinned_install_requires - - -def do_setup(pinned=False): +def do_setup(): """Perform the Airflow package setup.""" write_version() setup( - name='apache-airflow' + ("" if not pinned else "-pinned"), + name='apache-airflow', description='Programmatically author, schedule and monitor data pipelines', long_description=long_description, long_description_content_type='text/markdown', @@ -654,7 +612,7 @@ def do_setup(pinned=False): include_package_data=True, zip_safe=False, scripts=['airflow/bin/airflow'], - install_requires=get_install_requires(pinned=pinned), + install_requires=INSTALL_REQUIREMENTS, setup_requires=[ 'bowler', 'docutils>=0.14, <0.16' @@ -662,7 +620,7 @@ def do_setup(pinned=False): 'setuptools', 'wheel', ], - extras_require=get_extras_required(pinned=pinned), + extras_require=EXTRAS_REQUIREMENTS, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', @@ -699,12 +657,4 @@ def do_setup(pinned=False): "DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Airflow 1.10 " "will be the last release series to support Python 2\n" ) - pinned_requirements = False - if len(sys.argv) > 1 and sys.argv[1] == "pinned": - pinned_requirements = True - del sys.argv[1] - do_setup(pinned=pinned_requirements) - if sys.argv[1] == "--help": - print() - print("You can add 'pinned' as first argument to build pinned version of apache-airflow") - print() + do_setup() diff --git a/tests/bats/test_local_mounts.bats b/tests/bats/test_local_mounts.bats index 9c5de70fd6e86f..c45c1cc9762369 100644 --- a/tests/bats/test_local_mounts.bats +++ b/tests/bats/test_local_mounts.bats @@ -74,7 +74,7 @@ ${AIRFLOW_SOURCES}/logs:/root/airflow/logs:cached -v ${AIRFLOW_SOURCES}/pytest.ini:/opt/airflow/pytest.ini:cached -v -${AIRFLOW_SOURCES}/requirements.txt:/opt/airflow/requirements.txt:cached +${AIRFLOW_SOURCES}/requirements:/opt/airflow/requirements:cached -v ${AIRFLOW_SOURCES}/scripts:/opt/airflow/scripts:cached -v