diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index c03fcd9f81fb1..af33dd71fec5a 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -182,7 +182,7 @@ jobs: needs: [cancel-workflow-runs] strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8] image-type: [CI, PROD] fail-fast: true if: > @@ -209,14 +209,38 @@ jobs: uses: actions/checkout@v2 with: ref: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }} + - name: "Retrieve DEFAULTS from the _initialization.sh" + # We cannot "source" the script here because that would be a security problem (we cannot run + # any code that comes from the sources coming from the PR. Therefore we extract the + # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands + # Also 2.7 and 3.5 versions are not allowed to proceed on master + id: defaults + run: | + DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ + awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') + echo "::set-env name=DEFAULT_BRANCH::${DEFAULT_BRANCH}" + DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ + scripts/ci/libraries/_initialization.sh | \ + awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') + echo "::set-env name=DEFAULT_CONSTRAINTS_BRANCH::${DEFAULT_CONSTRAINTS_BRANCH}" + if [[ \ + ${DEFAULT_BRANCH} != "master" || \ + ( ${PYTHON_MAJOR_MINOR_VERSION} != "2.7" && ${PYTHON_MAJOR_MINOR_VERSION} != "3.5" ) \ + ]]; then + echo "::set-output name=proceed::true" + else + echo "::set-output name=proceed::false" + fi - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} ) to 'main-airflow' to use main scripts" uses: actions/checkout@v2 with: path: "main-airflow" + if: steps.defaults.outputs.proceed == 'true' - name: "Setup python" uses: actions/setup-python@v2 with: python-version: 3.6 + if: steps.defaults.outputs.proceed == 'true' - name: "Override 'scripts/ci' with the ${{ github.ref }} version so that the PR cannot override it." # We should not override those scripts which become part of the image as they will not be # changed in the image built - we should only override those that are executed to build @@ -224,20 +248,22 @@ jobs: run: | rm -rf "scripts/ci" mv "main-airflow/scripts/ci" "scripts" + if: steps.defaults.outputs.proceed == 'true' - name: "Free space" run: ./scripts/ci/tools/ci_free_space_on_ci.sh + if: steps.defaults.outputs.proceed == 'true' - name: "Build CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - if: matrix.image-type == 'CI' + if: matrix.image-type == 'CI' && steps.defaults.outputs.proceed == 'true' - name: "Push CI images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" run: ./scripts/ci/images/ci_push_ci_images.sh - if: matrix.image-type == 'CI' + if: matrix.image-type == 'CI' && steps.defaults.outputs.proceed == 'true' - name: "Build PROD images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh - if: matrix.image-type == 'PROD' + if: matrix.image-type == 'PROD' && steps.defaults.outputs.proceed == 'true' - name: "Push PROD images ${{ matrix.python-version }}:${{ github.event.workflow_run.id }}" run: ./scripts/ci/images/ci_push_production_images.sh - if: matrix.image-type == 'PROD' + if: matrix.image-type == 'PROD' && steps.defaults.outputs.proceed == 'true' - name: "Canceling the CI Build source workflow in case of failure!" if: cancelled() || failure() uses: potiuk/cancel-workflow-runs@v2 diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 42feaa4984749..2de09b91ad79a 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -526,6 +526,10 @@ function build_images::build_ci_image() { echo >&2 exit 1 fi + EXTRA_DOCKER_CI_BUILD_FLAGS=( + "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}" + ) + if [[ -n ${SPIN_PID=} ]]; then kill -HUP "${SPIN_PID}" || true wait "${SPIN_PID}" || true @@ -546,6 +550,7 @@ Docker building ${AIRFLOW_CI_IMAGE}. fi set +u docker build \ + "${EXTRA_DOCKER_CI_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ @@ -594,6 +599,7 @@ function build_images::prepare_prod_build() { else # When no airflow version/reference is specified, production image is built from local sources EXTRA_DOCKER_PROD_BUILD_FLAGS=( + "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}" ) fi if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" == "${PYTHON_MAJOR_MINOR_VERSION}" ]]; then diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index d63ac500cfe73..84db192637bbe 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -105,8 +105,9 @@ function initialization::initialize_base_variables() { # Determine current branch function initialization::initialize_branch_variables() { # Default branch used - this will be different in different branches - export DEFAULT_BRANCH="master" - export DEFAULT_CONSTRAINTS_BRANCH="constraints-master" + export DEFAULT_BRANCH=${DEFAULT_BRANCH="master"} + export DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH="constraints-master"} + readonly DEFAULT_BRANCH readonly DEFAULT_CONSTRAINTS_BRANCH # Default branch name for triggered builds is the one configured in default branch