Run Airflow package preparation in docker in breeze/CI (#15723)

Moved building airflow package to within the container similarly as we do with provider packages. This has the following advantages: * common environment used to build airflow * protection against leaking SECRET_* variables in CI in case third party packages are installed * specify --version-suffixes and renaming the packages according to destination (SVN/PyPI) automatically * no need to have node installed in CI runner * no need to have node installed in DockerHub * no need to install PIP/Python3 in DockerHub runner (currently Python2 is still default and it fails the build there) * always deleting egg-info and build before the build * cleaning up egg-info and build after the build Also following the way providers are released, the documentation is updated to change publishing Airflow using previously voted and renamed packages - the very same packages that were committed to SVN. This way anyone will be able to manually verify that the packages in SVN are the same as those published in SVN and there is no need to rebuild the packages when releasing them. (cherry picked from commit d627dfa)
apache · May 9, 2021 · 28652c8 · 28652c8
1 parent c4476d2
commit 28652c8
Show file tree

Hide file tree

Showing 18 changed files with 244 additions and 160 deletions.
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
@@ -409,8 +409,8 @@ jobs:
       DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }}
       FORCE_PULL_BASE_PYTHON_IMAGE: >
         ${{ needs.cancel-workflow-runs.sourceEvent == 'schedule' && 'true' || 'false' }}
-      VERSION_SUFFIX_FOR_PYPI: "dev"
-      VERSION_SUFFIX_FOR_SVN: "dev"
+      VERSION_SUFFIX_FOR_PYPI: ".dev0"
+      VERSION_SUFFIX_FOR_SVN: ".dev0"
     steps:
       - name: >
           Checkout [${{ needs.cancel-workflow-runs.outputs.sourceEvent }}]

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -502,8 +502,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
       AIRFLOW_EXTRAS: "all"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
-      VERSION_SUFFIX_FOR_PYPI: "dev"
-      VERSION_SUFFIX_FOR_SVN: "dev"
+      VERSION_SUFFIX_FOR_PYPI: ".dev0"
+      VERSION_SUFFIX_FOR_SVN: ".dev0"
       GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'master'
     steps:
@@ -527,7 +527,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         env:
           PACKAGE_FORMAT: "wheel"
       - name: "Prepare airflow package: wheel"
-        run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh
+        run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh
         env:
           PACKAGE_FORMAT: "wheel"
       - name: "Install and test provider packages and airflow via wheel files"
@@ -550,8 +550,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
       RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
       AIRFLOW_EXTRAS: "all"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
-      VERSION_SUFFIX_FOR_PYPI: "dev"
-      VERSION_SUFFIX_FOR_SVN: "dev"
+      VERSION_SUFFIX_FOR_PYPI: ".dev0"
+      VERSION_SUFFIX_FOR_SVN: ".dev0"
       GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
     if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'master'
     steps:
@@ -573,7 +573,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         env:
           PACKAGE_FORMAT: "sdist"
       - name: "Prepare airflow package: sdist"
-        run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh
+        run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh
         env:
           PACKAGE_FORMAT: "sdist"
       - name: "Install and test provider packages and airflow via sdist files"

diff --git a/BREEZE.rst b/BREEZE.rst
@@ -1716,11 +1716,19 @@ This is the current syntax for  `./breeze <./breeze>`_:
         Prepares airflow packages (sdist and wheel) in dist folder. Note that
         prepare-provider-packages command cleans up the dist folder, so if you want also
         to generate provider packages, make sure you run prepare-provider-packages first,
-        and prepare-airflow-packages second.
+        and prepare-airflow-packages second. You can specify optional
+        --version-suffix-for-svn flag to generate rc candidate packages to upload to SVN or
+        --version-suffix-for-pypi flag to generate rc candidates for PyPI packages. You can also
+        provide both suffixes in case you prepare alpha/beta versions. The packages are prepared in
+        dist folder
 
-        General form:
+        Examples:
 
-        'breeze prepare-airflow-packages
+        'breeze prepare-airflow-packages --package-format wheel' or
+        'breeze prepare-airflow-packages --version-suffix-for-svn rc1' or
+        'breeze prepare-airflow-packages --version-suffix-for-pypi rc1'
+        'breeze prepare-airflow-packages --version-suffix-for-pypi a1
+                                              --version-suffix-for-svn a1'
 
   Flags:
 
@@ -1734,6 +1742,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
           Default: both
 
+  -S, --version-suffix-for-pypi SUFFIX
+          Adds optional suffix to the version in the generated provider package. It can be used
+          to generate rc1/rc2 ... versions of the packages to be uploaded to PyPI.
+
+  -N, --version-suffix-for-svn SUFFIX
+          Adds optional suffix to the generated names of package. It can be used to generate
+          rc1/rc2 ... versions of the packages to be uploaded to SVN.
+
   -v, --verbose
           Show verbose information about executed docker, kind, kubectl, helm commands. Useful for
           debugging - when you run breeze with --verbose flags you will be able to see the commands

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -21,7 +21,7 @@ FROM ${PYTHON_BASE_IMAGE} as main
 SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
 
 ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
-ARG AIRFLOW_VERSION="2.0.0.dev0"
+ARG AIRFLOW_VERSION="2.1.0.dev0"
 # By increasing this number we can do force build of all dependencies
 ARG DEPENDENCIES_EPOCH_NUMBER="5"
 

diff --git a/breeze b/breeze
@@ -1868,14 +1868,23 @@ ${CMDNAME} prepare-airflow-packages [FLAGS]
       Prepares airflow packages (sdist and wheel) in dist folder. Note that
       prepare-provider-packages command cleans up the dist folder, so if you want also
       to generate provider packages, make sure you run prepare-provider-packages first,
-      and prepare-airflow-packages second.
+      and prepare-airflow-packages second. You can specify optional
+      --version-suffix-for-svn flag to generate rc candidate packages to upload to SVN or
+      --version-suffix-for-pypi flag to generate rc candidates for PyPI packages. You can also
+      provide both suffixes in case you prepare alpha/beta versions. The packages are prepared in
+      dist folder
 
-      General form:
+      Examples:
 
-      '${CMDNAME} prepare-airflow-packages
+      '${CMDNAME} prepare-airflow-packages --package-format wheel' or
+      '${CMDNAME} prepare-airflow-packages --version-suffix-for-svn rc1' or
+      '${CMDNAME} prepare-airflow-packages --version-suffix-for-pypi rc1'
+      '${CMDNAME} prepare-airflow-packages --version-suffix-for-pypi a1
+                                            --version-suffix-for-svn a1'
 
 Flags:
 $(breeze::flag_packages)
+$(breeze::flag_version_suffix)
 $(breeze::flag_verbosity)
 "
     readonly DETAILED_USAGE_PREPARE_AIRFLOW_PACKAGES
@@ -3497,7 +3506,7 @@ function breeze::run_breeze_command() {
         ;;
     perform_prepare_airflow_packages)
         docker_engine_resources::check_all_resources
-        build_airflow_packages::build_airflow_packages
+        runs::run_prepare_airflow_packages
         ;;
     perform_prepare_provider_packages)
         docker_engine_resources::check_all_resources

diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
@@ -56,7 +56,7 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag
     ```shell script
     # Set Version
     export VERSION=2.0.2rc3
-
+    export VERSION_SUFFIX=rc3
 
     # Set AIRFLOW_REPO_ROOT to the path of your git repo
     export AIRFLOW_REPO_ROOT=$(pwd)
@@ -80,42 +80,22 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag
 - Clean the checkout: the sdist step below will
 
     ```shell script
+    rm -rf dist/*
     git clean -fxd
     ```
 
 - Tarball the repo
 
     ```shell script
-    git archive --format=tar.gz ${VERSION} --prefix=apache-airflow-${VERSION}/ -o apache-airflow-${VERSION}-source.tar.gz
+    git archive --format=tar.gz ${VERSION} --prefix=apache-airflow-${VERSION}/ -o dist/apache-airflow-${VERSION}-source.tar.gz
     ```
 
 
-- Generate sdist
-
-    NOTE: Make sure your checkout is clean at this stage - any untracked or changed files will otherwise be included
-     in the file produced.
-
-    ```shell script
-    python setup.py compile_assets sdist bdist_wheel
-    ```
-
-- Rename the sdist
-
-    **Airflow 2+**:
-
-    ```shell script
-    mv dist/apache-airflow-${VERSION%rc?}.tar.gz apache-airflow-${VERSION}-bin.tar.gz
-    mv dist/apache_airflow-${VERSION%rc?}-py3-none-any.whl apache_airflow-${VERSION}-py3-none-any.whl
-    ```
-
 - Generate SHA512/ASC (If you have not generated a key yet, generate it by following instructions on http://www.apache.org/dev/openpgp.html#key-gen-generate-key)
 
-    **Airflow 2+**:
-
     ```shell script
-    ${AIRFLOW_REPO_ROOT}/dev/sign.sh apache-airflow-${VERSION}-source.tar.gz
-    ${AIRFLOW_REPO_ROOT}/dev/sign.sh apache-airflow-${VERSION}-bin.tar.gz
-    ${AIRFLOW_REPO_ROOT}/dev/sign.sh apache_airflow-${VERSION}-py3-none-any.whl
+    ./breeze prepare-airflow-packages --package-format both --version-suffix-for-svn "${VERSION_SUFFIX}"
+    ${AIRFLOW_REPO_ROOT}/dev/sign.sh dist/*
     ```
 
 - Tag & Push the latest constraints files. This pushes constraints with rc suffix (this is expected)!
@@ -128,20 +108,20 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag
 
 - Push the artifacts to ASF dev dist repo
 
-```
-# First clone the repo
-svn checkout https://dist.apache.org/repos/dist/dev/airflow airflow-dev
-
-# Create new folder for the release
-cd airflow-dev
-svn mkdir ${VERSION}
-
-# Move the artifacts to svn folder & commit
-mv ${AIRFLOW_REPO_ROOT}/apache{-,_}airflow-${VERSION}* ${VERSION}/
-cd ${VERSION}
-svn add *
-svn commit -m "Add artifacts for Airflow ${VERSION}"
-```
+    ```shell script
+    # First clone the repo
+    svn checkout https://dist.apache.org/repos/dist/dev/airflow airflow-dev
+
+    # Create new folder for the release
+    cd airflow-dev
+    svn mkdir ${VERSION}
+
+    # Move the artifacts to svn folder & commit
+    mv ${AIRFLOW_REPO_ROOT}/dist/* ${VERSION}/
+    cd ${VERSION}
+    svn add *
+    svn commit -m "Add artifacts for Airflow ${VERSION}"
+    ```
 
 ## Prepare PyPI convenience "snapshot" packages
 
@@ -156,7 +136,7 @@ To do this we need to
 - Build the package:
 
     ```shell script
-    python setup.py compile_assets egg_info --tag-build "$(sed -e "s/^[0-9.]*//" <<<"$VERSION")" sdist bdist_wheel
+    ./breeze prepare-airflow-package --version-suffix-for-pypi "${VERSION_SUFFIX}"
     ```
 
 - Verify the artifacts that would be uploaded:
@@ -201,23 +181,21 @@ pushed. If this did not happen - please login to DockerHub and check the status
 
 In case you need, you can also build and push the images manually:
 
-### Airflow 2+:
-
 ```shell script
-export VERSION_RC=<VERSION_HERE>
+export VERSION=<VERSION_HERE>
 export DOCKER_REPO=docker.io/apache/airflow
 for python_version in "3.6" "3.7" "3.8"
 (
-  export DOCKER_TAG=${VERSION_RC}-python${python_version}
+  export DOCKER_TAG=${VERSION}-python${python_version}
   ./scripts/ci/images/ci_build_dockerhub.sh
 )
 ```
 
-Once this succeeds you should push the "${VERSION_RC}" image:
+Once this succeeds you should push the "${VERSION}" image:
 
 ```shell script
-docker tag apache/airflow:${VERSION_RC}-python3.6 apache/airflow:${VERSION_RC}
-docker push apache/airflow:${VERSION_RC}
+docker tag apache/airflow:${VERSION}-python3.6 apache/airflow:${VERSION}
+docker push apache/airflow:${VERSION}
 ```
 
 This will wipe Breeze cache and docker-context-files in order to make sure the build is "clean". It
@@ -530,43 +508,56 @@ https://dist.apache.org/repos/dist/release/airflow/
 The best way of doing this is to svn cp between the two repos (this avoids having to upload the binaries again, and gives a clearer history in the svn commit logs):
 
 ```shell script
-# First clone the repo
+# GO to Airflow Sources first
+cd <YOUR_AIRFLOW_SOURCES>
+export AIRFLOW_SOURCES=$(pwd)
+
+# GO to Checked out DEV repo. Should be checked out before via:
+# svn checkout https://dist.apache.org/repos/dist/dev/airflow airflow-release
+cd <YOUR_AIFLOW_DEV_SVN>
+svn update
+export AIRFLOW_DEV_SVN=$(pwd)
+
+# GO to Checked out RELEASE repo. Should be checked out before via:
+# svn checkout https://dist.apache.org/repos/dist/dev/airflow airflow-release
+cd <YOUR_AIFLOW_RELEASE_SVN>
+svn update
+
 export RC=2.0.2rc5
 export VERSION=${RC/rc?/}
-svn checkout https://dist.apache.org/repos/dist/release/airflow airflow-release
 
 # Create new folder for the release
 cd airflow-release
-svn mkdir ${VERSION}
-cd ${VERSION}
+svn mkdir "${VERSION}"
+cd "${VERSION}"
 
 # Move the artifacts to svn folder & commit
-for f in ../../airflow-dev/$RC/*; do svn cp $f ${$(basename $f)/rc?/}; done
+for f in ${AIRFLOW_DEV_SVN}/$RC/*; do
+    svn cp "$f" "${$(basename $f)/rc?/}"
+    # Those will be used to upload to PyPI
+    cp "$f" "${AIRFLOW_SOURCES}/dist/${$(basename $f)/rc?/}"
+done
 svn commit -m "Release Airflow ${VERSION} from ${RC}"
 
 # Remove old release
-# http://www.apache.org/legal/release-policy.html#when-to-archive
+# See http://www.apache.org/legal/release-policy.html#when-to-archive
 cd ..
 export PREVIOUS_VERSION=2.0.2
-svn rm ${PREVIOUS_VERSION}
+svn rm "${PREVIOUS_VERSION}"
 svn commit -m "Remove old release: ${PREVIOUS_VERSION}"
 ```
 
 Verify that the packages appear in [airflow](https://dist.apache.org/repos/dist/release/airflow/)
 
 ## Prepare PyPI "release" packages
 
-At this point we release an official package:
-
-- Build the package:
-
-    ```shell script
-    python setup.py compile_assets sdist bdist_wheel
-    ```
+At this point we release an official package (they should be copied and renamed from the
+previously released RC candidates in "${AIRFLOW_SOURCES}/dist":
 
 - Verify the artifacts that would be uploaded:
 
     ```shell script
+    cd "${AIRFLOW_SOURCES}"
     twine check dist/*
     ```
 

diff --git a/...build_airflow/ci_build_airflow_package.sh → ...uild_airflow/ci_build_airflow_packages.sh b/...build_airflow/ci_build_airflow_package.sh → ...uild_airflow/ci_build_airflow_packages.sh
@@ -18,6 +18,6 @@
 # shellcheck source=scripts/ci/libraries/_script_init.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
 
-build_airflow_packages::build_airflow_packages
-
-cd "${AIRFLOW_SOURCES}/dist" || exit 1
+build_images::prepare_ci_build
+build_images::rebuild_ci_image_if_needed_with_group
+runs::run_prepare_airflow_packages
diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env
@@ -61,3 +61,4 @@ VERBOSE
 VERBOSE_COMMANDS
 VERSION_SUFFIX_FOR_PYPI
 VERSION_SUFFIX_FOR_SVN
+WHEEL_VERSION
diff --git a/scripts/ci/images/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh
@@ -55,8 +55,8 @@ if [[ ! "${DOCKER_TAG}" =~ ^[0-9].* ]]; then
     # All the packages: Airflow and providers will have a "dev" version suffix in the imaage that
     # is built from non-release tag. If this is not set, then building images from locally build
     # packages fails, because the packages with non-dev version are skipped (as they are alredy released)
-    export VERSION_SUFFIX_FOR_PYPI="dev"
-    export VERSION_SUFFIX_FOR_SVN="dev"
+    export VERSION_SUFFIX_FOR_PYPI=".dev0"
+    export VERSION_SUFFIX_FOR_SVN=".dev0"
     # Only build and push CI image for the nightly-master, v2-0-test branches
     # for tagged releases we build everything from PyPI, so we do not need CI images
     # For development images, we have to build all packages from current sources because we want to produce

diff --git a/scripts/ci/libraries/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh
@@ -36,8 +36,6 @@ readonly SCRIPTS_CI_DIR
 . "${LIBRARIES_DIR}"/_repeats.sh
 # shellcheck source=scripts/ci/libraries/_sanity_checks.sh
 . "${LIBRARIES_DIR}"/_sanity_checks.sh
-# shellcheck source=scripts/ci/libraries/_build_airflow_packages.sh
-. "${LIBRARIES_DIR}"/_build_airflow_packages.sh
 # shellcheck source=scripts/ci/libraries/_build_images.sh
 . "${LIBRARIES_DIR}"/_build_images.sh
 # shellcheck source=scripts/ci/libraries/_kind.sh