Update setup.py to get non-conflicting set of dependencies

This change upgrades setup.py to provide non-conflicting `pip check` valid set of constraints for CI image. Fixes #10854 Co-authored-by: Tomek Urbaszek <[email protected]>
apache · Nov 26, 2020 · c4b0460 · c4b0460
1 parent f1912d5
commit c4b0460
Show file tree

Hide file tree

Showing 5 changed files with 90 additions and 5 deletions.
diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
@@ -59,5 +59,5 @@ function build_ci_image_on_ci() {
     export CHECK_IMAGE_FOR_REBUILD="false"
 }
 
-
 build_ci_image_on_ci
+verify_prod_image_dependencies
diff --git a/scripts/ci/images/ci_wait_for_all_ci_images.sh b/scripts/ci/images/ci_wait_for_all_ci_images.sh
@@ -16,6 +16,18 @@
 # specific language governing permissions and limitations
 # under the License.
 export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
+
+function verify_ci_image_dependencies {
+    echo
+    echo "Checking if Airflow dependencies are non-conflicting in CI image."
+    echo
+
+    push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+    docker run --rm --entrypoint /bin/bash "${AIRFLOW_CI_IMAGE}" -c 'pip check'
+    return $?
+}
+
+
 echo
 echo "Airflow sources: ${AIRFLOW_SOURCES}"
 echo
@@ -54,3 +66,31 @@ do
     export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
     push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
 done
+
+echo
+echo "Verifying the images after pulling them"
+echo
+
+set +e
+
+verification_error="false"
+
+for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
+do
+    export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
+    echo
+    echo "Verifying ${AIRFLOW_CI_IMAGE_NAME}"
+    echo
+
+    if ! verify_ci_image_dependencies; then
+        verification_error="true"
+    fi
+done
+
+
+if [[ ${verification_error} == "true" ]]; then
+    echo
+    echo "ERROR! Some images did not pass verification!"
+    echo
+    exit 1
+fi
diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh b/scripts/ci/images/ci_wait_for_all_prod_images.sh
@@ -16,6 +16,19 @@
 # specific language governing permissions and limitations
 # under the License.
 export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
+
+function verify_prod_image_dependencies {
+    echo
+    echo "Checking if Airflow dependencies are non-conflicting in PROD image."
+    echo
+
+    push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+    docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'pip check'
+
+    return $?
+}
+
+
 echo
 echo "Airflow sources: ${AIRFLOW_SOURCES}"
 echo
@@ -56,3 +69,33 @@ do
     push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
     push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_BUILD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
 done
+
+
+echo
+echo "Verifying the images after pulling them"
+echo
+
+set +e
+
+verification_error="false"
+
+for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
+do
+    export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+
+    echo
+    echo "Verifying ${AIRFLOW_PROD_IMAGE_NAME}"
+    echo
+
+    if ! verify_prod_image_dependencies; then
+        verification_error="true"
+    fi
+done
+
+if [[ ${verification_error} == "true" ]]; then
+    echo
+    echo "ERROR! Some images did not pass verification!"
+    echo
+# TODO: we should enable that once constraints get updated and pip-checked in CI image
+#    exit 1
+fi
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -268,7 +268,7 @@ function push_pull_remove_images::wait_for_github_registry_image() {
     GITHUB_API_ENDPOINT="https://${GITHUB_REGISTRY}/v2/${github_repository_lowercase}"
     IMAGE_NAME="${1}"
     IMAGE_TAG=${2}
-    echo "Waiting for ${IMAGE_NAME}:${IMAGE_TAG} image"
+    echo "Waiting for ${GITHUB_REPOSITORY}/${IMAGE_NAME}:${IMAGE_TAG} image"
 
     GITHUB_API_CALL="${GITHUB_API_ENDPOINT}/${IMAGE_NAME}/manifests/${IMAGE_TAG}"
     while true; do

diff --git a/setup.py b/setup.py
@@ -163,7 +163,8 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
 # If you change this mark you should also change ./scripts/ci/check_order_setup.py
 # Start dependencies group
 amazon = [
-    'boto3>=1.12.0,<2.0.0',
+    'boto3>=1.16.0,<2.0.0',
+    'botocore>=1.18.0,<1.19.0',
     'watchtower~=0.7.3',
 ]
 apache_beam = [
@@ -188,7 +189,8 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'azure-mgmt-datalake-store>=0.5.0',
     'azure-mgmt-resource>=2.2.0',
     'azure-storage>=0.34.0, <0.37.0',
-    'azure-storage-blob<12.0',
+    'azure-storage-blob',
+    'azure-storage-common',
 ]
 cassandra = [
     'cassandra-driver>=3.13.0,<3.21.0',
@@ -463,7 +465,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'ipdb',
     'jira',
     'mongomock',
-    'moto>=1.3.16',
+    'moto',
     'parameterized',
     'paramiko',
     'pipdeptree',