diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh index e2637c3aee5341..650566529be271 100755 --- a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh +++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh @@ -59,5 +59,5 @@ function build_ci_image_on_ci() { export CHECK_IMAGE_FOR_REBUILD="false" } - build_ci_image_on_ci +verify_prod_image_dependencies diff --git a/scripts/ci/images/ci_wait_for_all_ci_images.sh b/scripts/ci/images/ci_wait_for_all_ci_images.sh index edb6b294640e74..507286f3185b08 100755 --- a/scripts/ci/images/ci_wait_for_all_ci_images.sh +++ b/scripts/ci/images/ci_wait_for_all_ci_images.sh @@ -16,6 +16,18 @@ # specific language governing permissions and limitations # under the License. export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}" + +function verify_ci_image_dependencies { + echo + echo "Checking if Airflow dependencies are non-conflicting in CI image." + echo + + push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + docker run --rm --entrypoint /bin/bash "${AIRFLOW_CI_IMAGE}" -c 'pip check' + return $? +} + + echo echo "Airflow sources: ${AIRFLOW_SOURCES}" echo @@ -54,3 +66,31 @@ do export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" done + +echo +echo "Verifying the images after pulling them" +echo + +set +e + +verification_error="false" + +for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} +do + export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" + echo + echo "Verifying ${AIRFLOW_CI_IMAGE_NAME}" + echo + + if ! verify_ci_image_dependencies; then + verification_error="true" + fi +done + + +if [[ ${verification_error} == "true" ]]; then + echo + echo "ERROR! Some images did not pass verification!" + echo + exit 1 +fi diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh b/scripts/ci/images/ci_wait_for_all_prod_images.sh index 66196c3257eba2..3a8d06535391c4 100755 --- a/scripts/ci/images/ci_wait_for_all_prod_images.sh +++ b/scripts/ci/images/ci_wait_for_all_prod_images.sh @@ -16,6 +16,19 @@ # specific language governing permissions and limitations # under the License. export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}" + +function verify_prod_image_dependencies { + echo + echo "Checking if Airflow dependencies are non-conflicting in PROD image." + echo + + push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" + docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'pip check' + + return $? +} + + echo echo "Airflow sources: ${AIRFLOW_SOURCES}" echo @@ -56,3 +69,33 @@ do push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_BUILD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" done + + +echo +echo "Verifying the images after pulling them" +echo + +set +e + +verification_error="false" + +for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} +do + export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}" + + echo + echo "Verifying ${AIRFLOW_PROD_IMAGE_NAME}" + echo + + if ! verify_prod_image_dependencies; then + verification_error="true" + fi +done + +if [[ ${verification_error} == "true" ]]; then + echo + echo "ERROR! Some images did not pass verification!" + echo +# TODO: we should enable that once constraints get updated and pip-checked in CI image +# exit 1 +fi diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh index 9d4e23fbdbfac8..54fff0571a979a 100644 --- a/scripts/ci/libraries/_push_pull_remove_images.sh +++ b/scripts/ci/libraries/_push_pull_remove_images.sh @@ -268,7 +268,7 @@ function push_pull_remove_images::wait_for_github_registry_image() { GITHUB_API_ENDPOINT="https://${GITHUB_REGISTRY}/v2/${github_repository_lowercase}" IMAGE_NAME="${1}" IMAGE_TAG=${2} - echo "Waiting for ${IMAGE_NAME}:${IMAGE_TAG} image" + echo "Waiting for ${GITHUB_REPOSITORY}/${IMAGE_NAME}:${IMAGE_TAG} image" GITHUB_API_CALL="${GITHUB_API_ENDPOINT}/${IMAGE_NAME}/manifests/${IMAGE_TAG}" while true; do diff --git a/setup.py b/setup.py index eae499ba86d3b3..c6e4c323cb7b99 100644 --- a/setup.py +++ b/setup.py @@ -163,7 +163,8 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version # If you change this mark you should also change ./scripts/ci/check_order_setup.py # Start dependencies group amazon = [ - 'boto3>=1.12.0,<2.0.0', + 'boto3>=1.16.0,<2.0.0', + 'botocore>=1.18.0,<1.19.0', 'watchtower~=0.7.3', ] apache_beam = [ @@ -188,7 +189,8 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'azure-mgmt-datalake-store>=0.5.0', 'azure-mgmt-resource>=2.2.0', 'azure-storage>=0.34.0, <0.37.0', - 'azure-storage-blob<12.0', + 'azure-storage-blob', + 'azure-storage-common', ] cassandra = [ 'cassandra-driver>=3.13.0,<3.21.0', @@ -463,7 +465,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'ipdb', 'jira', 'mongomock', - 'moto>=1.3.16', + 'moto', 'parameterized', 'paramiko', 'pipdeptree',