diff --git a/BREEZE.rst b/BREEZE.rst index f91b598fe03796..095fe1b30a5ffe 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1355,6 +1355,7 @@ This is the current syntax for `./breeze <./breeze>`_: If you use this flag, automatically --github-registry is enabled. + Default: latest. -v, --verbose @@ -1508,6 +1509,7 @@ This is the current syntax for `./breeze <./breeze>`_: If you use this flag, automatically --github-registry is enabled. + Default: latest. -v, --verbose @@ -2276,6 +2278,7 @@ This is the current syntax for `./breeze <./breeze>`_: If you use this flag, automatically --github-registry is enabled. + Default: latest. **************************************************************************************************** diff --git a/CI.rst b/CI.rst index f4b5294cd04580..fac9f0fdbe3341 100644 --- a/CI.rst +++ b/CI.rst @@ -253,7 +253,7 @@ You can use those variables when you try to reproduce the build locally. | Image build variables | +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+ | ``UPGRADE_TO_LATEST_CONSTRAINTS`` | false | false | false | Determines whether the build should | -| | | | (x) | attempt to eagerly upgrade all | +| | | | (x) | attempt to upgrade all | | | | | | PIP dependencies to latest ones matching | | | | | | ``setup.py`` limits. This tries to replicate | | | | | | the situation of "fresh" user who just installs | diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 61883e7ab7096c..0c1c9c12a1e580 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -321,7 +321,7 @@ Step 4: Prepare PR the "full tests needed" label is set for your PR. Additional check is set that prevents from accidental merging of the request until full matrix of tests succeeds for the PR. - * when your change has "upgrade to latest dependencies" label set, constraints will be automatically + * when your change has "upgrade to newer dependencies" label set, constraints will be automatically upgraded to latest constraints matching your setup.py. This is useful in case you want to force upgrade to a latest version of dependencies. You can ask committers to set the label for you when you need it in your PR. diff --git a/breeze b/breeze index fe8f0386564262..6f73ad706ca0e6 100755 --- a/breeze +++ b/breeze @@ -1078,6 +1078,7 @@ function breeze::parse_arguments() { echo echo "Force pulling the image, using github registry and skip mounting local sources." echo "This is in order to get the exact same version as used in CI environment for SHA/RUN_ID!." + echo "You can specify --skip-mounting-local-sources to not mount local sources. " echo export FORCE_PULL_IMAGES="true" export USE_GITHUB_REGISTRY="true" @@ -2385,6 +2386,7 @@ function breeze::flag_pull_push_docker_images() { If you use this flag, automatically --github-registry is enabled. + Default: ${_breeze_default_github_image_id:=}. " diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh index e2637c3aee5341..4d2a1ce5922c79 100755 --- a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh +++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh @@ -59,5 +59,4 @@ function build_ci_image_on_ci() { export CHECK_IMAGE_FOR_REBUILD="false" } - build_ci_image_on_ci diff --git a/scripts/ci/images/ci_wait_for_all_ci_images.sh b/scripts/ci/images/ci_wait_for_all_ci_images.sh index edb6b294640e74..2451a880fd58aa 100755 --- a/scripts/ci/images/ci_wait_for_all_ci_images.sh +++ b/scripts/ci/images/ci_wait_for_all_ci_images.sh @@ -15,42 +15,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}" echo -echo "Airflow sources: ${AIRFLOW_SOURCES}" +echo "Waiting for all CI images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}" echo -if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then - echo - echo "This script should not be called" - echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!" - echo - echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}" - echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}" - echo - exit 1 -fi - -echo -echo "Waiting for all images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}" -echo - -echo -echo "Check if jq is installed" -echo -command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1) - -echo -echo "The jq version $(jq --version)" -echo - -# shellcheck source=scripts/ci/libraries/_all_libs.sh -source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh" - -initialization::initialize_common_environment - for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} do - export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" - push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + export PYTHON_MAJOR_MINOR_VERSION + "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_ci_image.sh" done diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh b/scripts/ci/images/ci_wait_for_all_prod_images.sh index 66196c3257eba2..25bfd7c42cf999 100755 --- a/scripts/ci/images/ci_wait_for_all_prod_images.sh +++ b/scripts/ci/images/ci_wait_for_all_prod_images.sh @@ -15,44 +15,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}" echo -echo "Airflow sources: ${AIRFLOW_SOURCES}" +echo "Waiting for all PROD images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}" echo -if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then - echo - echo "This script should not be called" - echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!" - echo - echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}" - echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}" - echo - exit 1 -fi - -echo -echo "Waiting for all images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}" -echo - -echo -echo "Check if jq is installed" -echo -command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1) - -echo -echo "The jq version $(jq --version)" -echo - -# shellcheck source=scripts/ci/libraries/_all_libs.sh -source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh" - -initialization::initialize_common_environment - for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} do - export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}" - export AIRFLOW_PROD_BUILD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-build" - push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_BUILD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + export PYTHON_MAJOR_MINOR_VERSION + "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_prod_image.sh" done diff --git a/scripts/ci/images/ci_wait_for_ci_image.sh b/scripts/ci/images/ci_wait_for_ci_image.sh new file mode 100755 index 00000000000000..2c0bdf230c6b46 --- /dev/null +++ b/scripts/ci/images/ci_wait_for_ci_image.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +function verify_ci_image_dependencies { + echo + echo "Checking if Airflow dependencies are non-conflicting in CI image." + echo + + push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" \ + "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + + # TODO: remove after we have it fully working + docker run --rm --entrypoint /bin/bash "${AIRFLOW_CI_IMAGE}" -c 'pip check' || true +} + +push_pull_remove_images::check_if_github_registry_wait_for_image_enabled + +push_pull_remove_images::check_if_jq_installed + +build_image::login_to_github_registry_if_needed + +export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" + +echo +echo "Waiting for image to appear: ${AIRFLOW_CI_IMAGE_NAME}" +echo + +push_pull_remove_images::wait_for_github_registry_image \ + "${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + +echo +echo "Verifying the ${AIRFLOW_CI_IMAGE_NAME} image after pulling it" +echo + +verify_ci_image_dependencies diff --git a/scripts/ci/images/ci_wait_for_prod_image.sh b/scripts/ci/images/ci_wait_for_prod_image.sh new file mode 100755 index 00000000000000..e53aec130b8938 --- /dev/null +++ b/scripts/ci/images/ci_wait_for_prod_image.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" + +function verify_prod_image_dependencies { + echo + echo "Checking if Airflow dependencies are non-conflicting in PROD image." + echo + + push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" \ + "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + + # TODO: remove the | true after we fixed pip check for prod image + docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'pip check' || true +} + +push_pull_remove_images::check_if_github_registry_wait_for_image_enabled + +push_pull_remove_images::check_if_jq_installed + +build_image::login_to_github_registry_if_needed + +export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}" + +echo +echo "Waiting for image to appear: ${AIRFLOW_PROD_IMAGE_NAME}" +echo + +push_pull_remove_images::wait_for_github_registry_image \ + "${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + +echo +echo "Verifying the ${AIRFLOW_PROD_IMAGE_NAME} image after pulling it" +echo + +verify_prod_image_dependencies diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 5bd2d06b0aa4d8..8de58db0124888 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -346,13 +346,19 @@ function build_images::get_docker_image_names() { # File that is touched when the CI image is built for the first time locally export BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}" + + # GitHub Registry names must be lowercase :( + github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')" + export GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}" + export GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}-build" + export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" + + export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_CI_BASE_TAG}" + export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" } -# Prepares all variables needed by the CI build. Depending on the configuration used (python version -# DockerHub user etc. the variables are set so that other functions can use those variables. -function build_images::prepare_ci_build() { - export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest" - export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest" +# If GitHub Registry is used, login to the registry using GITHUB_USERNAME and GITHUB_TOKEN +function build_image::login_to_github_registry_if_needed() { if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then if [[ -n ${GITHUB_TOKEN=} ]]; then echo "${GITHUB_TOKEN}" | docker login \ @@ -360,11 +366,15 @@ function build_images::prepare_ci_build() { --password-stdin \ "${GITHUB_REGISTRY}" fi - # GitHub Registry names must be lowercase :( - github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')" - export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_CI_BASE_TAG}" - export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" fi + +} + +# Prepares all variables needed by the CI build. Depending on the configuration used (python version +# DockerHub user etc. the variables are set so that other functions can use those variables. +function build_images::prepare_ci_build() { + export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest" + export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest" export THE_IMAGE_TYPE="CI" export IMAGE_DESCRIPTION="Airflow CI" @@ -375,6 +385,7 @@ function build_images::prepare_ci_build() { export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}" readonly AIRFLOW_IMAGE + build_image::login_to_github_registry_if_needed sanity_checks::go_to_airflow_sources permissions::fix_group_permissions } @@ -662,19 +673,7 @@ function build_images::prepare_prod_build() { export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}" readonly AIRFLOW_IMAGE - if [[ ${USE_GITHUB_REGISTRY="false"} == "true" ]]; then - if [[ -n ${GITHUB_TOKEN=} ]]; then - echo "${GITHUB_TOKEN}" | docker login \ - --username "${GITHUB_USERNAME}" \ - --password-stdin \ - "${GITHUB_REGISTRY}" - fi - # GitHub Registry names must be lowercase :( - github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')" - export GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}" - export GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}-build" - export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster" - fi + build_image::login_to_github_registry_if_needed AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="${BRANCH_NAME}" sanity_checks::go_to_airflow_sources diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh index 7c65db19c7ad5a..216e025b5a11e1 100644 --- a/scripts/ci/libraries/_push_pull_remove_images.sh +++ b/scripts/ci/libraries/_push_pull_remove_images.sh @@ -264,13 +264,18 @@ function push_pull_remove_images::push_prod_images() { # waits for an image to be available in the github registry function push_pull_remove_images::wait_for_github_registry_image() { + local github_repository_lowercase github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')" - GITHUB_API_ENDPOINT="https://${GITHUB_REGISTRY}/v2/${github_repository_lowercase}" - IMAGE_NAME="${1}" - IMAGE_TAG=${2} - echo "Waiting for ${IMAGE_NAME}:${IMAGE_TAG} image" + local github_api_endpoint + github_api_endpoint="https://${GITHUB_REGISTRY}/v2/${github_repository_lowercase}" + local image_name_in_github_registry="${1}" + local image_tag_in_github_registry=${2} - GITHUB_API_CALL="${GITHUB_API_ENDPOINT}/${IMAGE_NAME}/manifests/${IMAGE_TAG}" + echo + echo "Waiting for ${GITHUB_REPOSITORY}/${image_name_in_github_registry}:${image_tag_in_github_registry} image" + echo + + GITHUB_API_CALL="${github_api_endpoint}/${image_name_in_github_registry}/manifests/${image_tag_in_github_registry}" while true; do curl -X GET "${GITHUB_API_CALL}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" 2>/dev/null > "${OUTPUT_LOG}" local digest @@ -282,6 +287,33 @@ function push_pull_remove_images::wait_for_github_registry_image() { fi sleep 10 done - verbosity::print_info "Found ${IMAGE_NAME}:${IMAGE_TAG} image" + verbosity::print_info "Found ${image_name_in_github_registry}:${image_tag_in_github_registry} image" verbosity::print_info "Digest: '${digest}'" } + +function push_pull_remove_images::check_if_github_registry_wait_for_image_enabled() { + if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then + echo + echo "This script should not be called" + echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!" + echo + echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}" + echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}" + echo + exit 1 + else + echo + echo "Both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE are set to true. Good!" + fi +} + +function push_pull_remove_images::check_if_jq_installed() { + echo + echo "Check if jq is installed" + echo + command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1) + + echo + echo "The jq version $(jq --version)" + echo +} diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index c87ec41f4b9d8b..8696d561a5a91c 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -42,14 +42,14 @@ else FULL_TESTS_NEEDED_LABEL="false" fi -if [[ ${PR_LABELS=} == *"upgrade to latest dependencies"* ]]; then +if [[ ${PR_LABELS=} == *"upgrade to newer dependencies"* ]]; then echo - echo "Found the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies''" + echo "Found the right PR labels in '${PR_LABELS=}': 'upgrade to newer dependencies''" echo UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="true" else echo - echo "Did not find the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies'" + echo "Did not find the right PR labels in '${PR_LABELS=}': 'upgrade to newer dependencies'" echo UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="false" fi diff --git a/setup.py b/setup.py index f5f2a5384f17ea..83e8a98b1eaf2d 100644 --- a/setup.py +++ b/setup.py @@ -182,11 +182,13 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'atlasclient>=0.1.2', ] aws = [ - 'boto3~=1.10', + 'boto3~=1.10,<1.11', # required by snowflake ] azure_blob_storage = [ 'azure-storage>=0.34.0, <0.37.0', - 'azure-storage-blob<12.0', + 'azure-storage-blob<12.0.0;python_version<"3.6"', + 'azure-storage-blob;python_version>="3.6"', + 'azure-storage-common', ] azure_container_instances = [ 'azure-mgmt-containerinstance>=1.5.0,<2' @@ -198,6 +200,7 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'azure-datalake-store>=0.0.45' 'azure-mgmt-datalake-store>=0.5.0', 'azure-mgmt-resource>=2.2.0', + 'cffi<1.14.0;python_version<"3.0"' ] azure_secrets = [ 'azure-identity>=1.3.1', @@ -207,7 +210,8 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'cassandra-driver>=3.13.0,<3.21.0', ] celery = [ - 'celery~=4.3', + 'celery~=4.3;python_version>="3.0"', + 'celery==4.3.1;python_version<"3.0"', 'flower>=0.7.3, <1.0', 'kombu==4.6.3;python_version<"3.0"', 'tornado>=4.2.0, <6.0', # Dep of flower. Pin to a version that works on Py3.5.2 @@ -222,7 +226,8 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): crypto = [ # Cryptography 3.2 for python 2.7 is broken # https://github.com/pyca/cryptography/issues/5359#issuecomment-727622403 - 'cryptography>=0.9.3,<3.2; python_version<"3.0"', + # Snowflake requires <3.0 + 'cryptography>=0.9.3,<3.0; python_version<"3.0"', 'cryptography>=0.9.3;python_version>="3.0"', ] dask = [ @@ -260,7 +265,8 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'requests-oauthlib==1.1.0', ] gcp = [ - 'PyOpenSSL', + 'PyOpenSSL<20.0.0;python_version<"3.0"', + 'PyOpenSSL;python_version>="3.0"', 'google-api-python-client>=1.6.0, <2.0.0', 'google-auth>=1.0.0, <2.0.0', 'google-auth-httplib2>=0.0.1', @@ -306,7 +312,8 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): kerberos = [ 'pykerberos>=1.1.13', 'requests_kerberos>=0.10.0', - 'thrift_sasl>=0.2.0', + 'thrift_sasl>=0.2.0,<0.4.1;python_version<"3.0"', + 'thrift_sasl>=0.2.0;python_version>="3.0"', ] kubernetes = [ 'cryptography>=2.0.0', @@ -336,7 +343,9 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'papermill[all]>=1.0.0', 'nteract-scrapbook[all]>=0.2.1', 'pyarrow<1.0.0', - 'fsspec<0.8.0;python_version=="3.5"' + 'fsspec<0.8.0;python_version=="3.5"', + 'black==20.8b0;python_version>="3.6"' # we need to limit black version as we have click < 7 + ] password = [ 'bcrypt>=2.0.0', @@ -355,7 +364,7 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'qds-sdk>=1.10.4', ] rabbitmq = [ - 'amqp', + 'amqp<5.0.0', ] redis = [ 'redis~=3.2', @@ -378,6 +387,7 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): ] slack = [ 'slackclient>=1.0.0,<2.0.0', + 'websocket-client<0.55.0' ] snowflake = [ 'snowflake-connector-python>=1.5.2', @@ -421,11 +431,14 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'click==6.7', 'contextdecorator;python_version<"3.4"', 'coverage', + 'docutils>=0.14, <0.16', + 'ecdsa<0.15', # Required for moto 1.3.14 'flake8>=3.6.0', 'flake8-colors', 'flaky', 'freezegun', 'gitpython', + 'idna<2.9', # Required for moto 1.3.14 'importlib-metadata~=2.0; python_version<"3.8"', 'ipdb', 'jira', @@ -436,14 +449,15 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'packaging', 'parameterized', 'paramiko', + 'pipdeptree', 'pre-commit', + 'pyrsistent<=0.16.0;python_version<"3.0"', + 'pyrsistent;python_version>="3.0"', 'pysftp', 'pytest<6.0.0', # FIXME: pylint complaining for pytest.mark.* on v6.0 'pytest-cov', 'pytest-instafail', - 'pytest-rerunfailures', 'pytest-timeouts', - 'pytest-xdist', 'pywinrm', 'qds-sdk>=1.9.6', 'requests_mock', @@ -590,6 +604,8 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'colorlog==4.0.2', 'configparser>=3.5.0, <3.6.0', 'croniter>=0.3.17, <0.4', + 'cryptography>=0.9.3,<3.0; python_version<"3.0"', # required by snowflake + 'cryptography>=0.9.3;python_version>="3.0"', 'dill>=0.2.2, <0.4', 'email-validator', 'enum34~=1.1.6;python_version<"3.4"', @@ -606,11 +622,12 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'graphviz>=0.12', 'gunicorn>=19.5.0, <21.0', 'importlib-metadata~=2.0; python_version<"3.8"', + 'importlib_resources~=1.4', 'iso8601>=0.1.12', 'jinja2>=2.10.1, <2.12.0', 'json-merge-patch==0.2', 'jsonschema~=3.0', - 'lazy_object_proxy~=1.3', + 'lazy_object_proxy<1.5.0', # Required to keep pip-check happy with astroid 'markdown>=2.5.2, <3.0', 'marshmallow-sqlalchemy>=0.16.1, <0.24.0;python_version>="3.6"', 'marshmallow-sqlalchemy>=0.16.1, <0.19.0;python_version<"3.6"', @@ -624,14 +641,15 @@ def write_version(filename=os.path.join(*[my_dir, "airflow", "git_version"])): 'python-dateutil>=2.3, <3', 'python-nvd3~=0.15.0', 'python-slugify>=3.0.0,<5.0', - 'requests>=2.20.0, <3', + 'requests>=2.20.0, <2.23.0;python_version<"3.0"', # Required to keep snowflake happy + 'requests>=2.20.0, <2.24.0;python_version>="3.0"', # Required to keep snowflake happy 'setproctitle>=1.1.8, <2', 'sqlalchemy~=1.3', 'sqlalchemy_jsonfield==0.8.0;python_version<"3.5"', 'sqlalchemy_jsonfield~=0.9;python_version>="3.5"', 'tabulate>=0.7.5, <0.9', 'tenacity==4.12.0', - 'thrift>=0.9.2', + 'thrift>=0.11.0', 'typing;python_version<"3.5"', 'typing-extensions>=3.7.4;python_version<"3.8"', 'tzlocal>=1.4,<2.0.0',