From 34f83be1e4cd249c0884ed1a51ebd1399dd6aec0 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Fri, 23 Apr 2021 17:16:22 +0100 Subject: [PATCH 1/7] Test out pull_request_target --- .github/actions/cancel-workflow-runs | 1 - .github/workflows/build-images.yml | 337 +++++++++++++++++++++++++++ .github/workflows/ci.yml | 15 +- .gitmodules | 3 - 4 files changed, 343 insertions(+), 13 deletions(-) delete mode 160000 .github/actions/cancel-workflow-runs create mode 100644 .github/workflows/build-images.yml diff --git a/.github/actions/cancel-workflow-runs b/.github/actions/cancel-workflow-runs deleted file mode 160000 index 953e057dc81d34..00000000000000 --- a/.github/actions/cancel-workflow-runs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 953e057dc81d3458935a18d1184c386b0f6b5738 diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml new file mode 100644 index 00000000000000..cf71c8357fd422 --- /dev/null +++ b/.github/workflows/build-images.yml @@ -0,0 +1,337 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: "Build Images" +on: # yamllint disable-line rule:truthy + schedule: + - cron: '28 0 * * *' + pull_request_target: + push: + branches: ['main', 'master', 'v1-10-test', 'v1-10-stable', 'v2-0-test'] +env: + MOUNT_SELECTED_LOCAL_SOURCES: "false" + FORCE_ANSWER_TO_QUESTIONS: "yes" + FORCE_PULL_IMAGES: "false" + CHECK_IMAGE_FOR_REBUILD: "true" + SKIP_CHECK_REMOTE_IMAGE: "true" + DB_RESET: "true" + VERBOSE: "true" + USE_GITHUB_REGISTRY: "true" + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_USERNAME: ${{ github.actor }} + # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the + # Airflow one is going to be used + CONSTRAINTS_GITHUB_REPOSITORY: >- + ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && + secrets.CONSTRAINTS_GITHUB_REPOSITORY || github.repository }} + # This token is WRITE one - pull_request_target type of events always have the WRITE token + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # This token should not be empty in pull_request_target type of event. + CONTAINER_REGISTRY_TOKEN: ${{ secrets.PAT_CR }} + GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" + GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" + INSTALL_PROVIDERS_FROM_SOURCES: "true" + GITHUB_REGISTRY: ${{ secrets.OVERRIDE_GITHUB_REGISTRY }} + TARGET_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + +concurrency: + group: build-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + + build-info: + timeout-minutes: 10 + name: "Build Info" + runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }} + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + targetBranch: ${{ github.event.pull_request.base.ref }} + pullRequestLabels: "${{ toJSON(github.event.pull_request.labels.*.name) }}" + outputs: + runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }} + pythonVersions: "${{ steps.selective-checks.python-versions }}" + upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} + allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} + defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} + run-tests: ${{ steps.selective-checks.outputs.run-tests }} + run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} + image-build: ${{ steps.dynamic-outputs.outputs.image-build }} + cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }} + targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }} + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: env + run: printenv + env: + dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }} + - name: Selective checks + id: selective-checks + env: + PR_LABELS: ${{ env.pullRequestLabels }} + run: | + if [[ ${GITHUB_EVENT_NAME} == "pull_request_target" ]]; then + # Run selective checks + ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" + else + # Run all checks + ./scripts/ci/selective_ci_checks.sh + fi + - name: Compute dynamic outputs + id: dynamic-outputs + run: | + set -x + if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then + echo "::set-output name=targetBranch::${targetBranch}" + else + # Direct push to branch, or scheduled build + echo "::set-output name=targetBranch::${GITHUB_REF#refs/heads/}" + fi + + if [[ "${{ github.event_name }}" == 'schedule' ]]; then + echo "::set-output name=cacheDirective::disabled" + else + echo "::set-output name=cacheDirective::pulled" + fi + + if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" && "$BUILD_IMAGES_OVERRIDE" != "false" ]]; then + echo "::set-output name=image-build::true" + else + echo "::set-output name=image-build::false" + fi + env: + SELECTIVE_CHECKS_IMAGE_BUILD: ${{ steps.selective-checks.outputs.image-build }} + BUILD_IMAGES_OVERRIDE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE }} + + build-ci-images: + timeout-minutes: 80 + name: "Build CI images ${{matrix.python-version}}" + runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + needs: [build-info] + strategy: + matrix: + # We need to attempt to build all possible versions here because pull_request_target + # event is run from master for both master and v1-10-tests + python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} + fail-fast: true + if: needs.build-info.outputs.image-build == 'true' + env: + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} + BACKEND: postgres + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} + CONTINUE_ON_PIP_CHECK_FAILURE: "true" + DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} + FORCE_PULL_BASE_PYTHON_IMAGE: > + ${{ github.event_name == 'schedule' && 'true' || 'false' }} + outputs: ${{toJSON(needs.build-info.outputs) }} + steps: + - name: Set envs + # Can't refer to "global" env to set a var in a job's env context + run: | + echo "GITHUB_REGISTRY_PUSH_IMAGE_TAG=${TARGET_COMMIT_SHA}" >> "$GITHUB_ENV" + - uses: actions/checkout@v2 + with: + ref: ${{ env.TARGET_COMMIT_SHA }} + persist-credentials: false + submodules: recursive + - name: "Retrieve DEFAULTS from the _initialization.sh" + # We cannot "source" the script here because that would be a security problem (we cannot run + # any code that comes from the sources coming from the PR. Therefore we extract the + # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands + # Also 2.7 and 3.5 versions are not allowed to proceed on master + id: defaults + run: | + DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ + awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') + echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV + DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ + scripts/ci/libraries/_initialization.sh | \ + awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') + echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV + if [[ ${DEFAULT_BRANCH} != "v1-10-test" && \ + ( ${PYTHON_MAJOR_MINOR_VERSION} == "2.7" || ${PYTHON_MAJOR_MINOR_VERSION} == "3.5" ) \ + ]]; then + echo "::set-output name=proceed::false" + else + echo "::set-output name=proceed::true" + fi + - name: > + Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder + to use ci/scripts from there. + uses: actions/checkout@v2 + with: + path: "main-airflow" + ref: "${{ needs.build-info.outputs.targetBranch }}" + persist-credentials: false + submodules: recursive + if: steps.defaults.outputs.proceed == 'true' + - name: "Setup python" + uses: actions/setup-python@v2 + with: + python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + if: steps.defaults.outputs.proceed == 'true' + - name: > + Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch + so that the PR does not override it + # We should not override those scripts which become part of the image as they will not be + # changed in the image built - we should only override those that are executed to build + # the image. + run: | + rm -rf "scripts/ci" + mv "main-airflow/scripts/ci" "scripts" + if: steps.defaults.outputs.proceed == 'true' + - name: "Free space" + run: ./scripts/ci/tools/ci_free_space_on_ci.sh + if: steps.defaults.outputs.proceed == 'true' + - name: "Build CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh + if: steps.defaults.outputs.proceed == 'true' + - name: "Push CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + run: ./scripts/ci/images/ci_push_ci_images.sh + if: steps.defaults.outputs.proceed == 'true' + + build-prod-images: + timeout-minutes: 80 + name: "Build PROD images ${{matrix.python-version}}" + runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + needs: [build-info, build-ci-images] + strategy: + matrix: + # We need to attempt to build all possible versions here because pull_request_target + # event is run from master for both master and v1-10-tests + python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} + fail-fast: true + if: needs.build-info.outputs.image-build == 'true' + env: + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} + BACKEND: postgres + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} + FORCE_PULL_BASE_PYTHON_IMAGE: > + ${{ github.event_name == 'schedule' && 'true' || 'false' }} + VERSION_SUFFIX_FOR_PYPI: ".dev0" + steps: + - name: Set envs + run: | + echo "GITHUB_REGISTRY_PUSH_IMAGE_TAG=${TARGET_COMMIT_SHA}" >> "$GITHUB_ENV" + echo "GITHUB_REGISTRY_PULL_IMAGE_TAG=${TARGET_COMMIT_SHA}" >> "$GITHUB_ENV" + - uses: actions/checkout@v2 + with: + ref: ${{ env.TARGET_COMMIT_SHA }} + persist-credentials: false + submodules: recursive + - name: "Retrieve DEFAULTS from the _initialization.sh" + # We cannot "source" the script here because that would be a security problem (we cannot run + # any code that comes from the sources coming from the PR. Therefore we extract the + # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH via custom grep/awk/sed commands + # Also 2.7 and 3.5 versions are not allowed to proceed on master + id: defaults + run: | + DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ + awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') + echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV + DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ + scripts/ci/libraries/_initialization.sh | \ + awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') + echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV + if [[ ${DEFAULT_BRANCH} != "v1-10-test" && \ + ( ${PYTHON_MAJOR_MINOR_VERSION} == "2.7" || ${PYTHON_MAJOR_MINOR_VERSION} == "3.5" ) \ + ]]; then + echo "::set-output name=proceed::false" + else + echo "::set-output name=proceed::true" + fi + - name: > + Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder + to use ci/scripts from there. + uses: actions/checkout@v2 + with: + path: "main-airflow" + ref: "${{ needs.build-info.outputs.targetBranch }}" + persist-credentials: false + submodules: recursive + if: steps.defaults.outputs.proceed == 'true' + - name: "Setup python" + uses: actions/setup-python@v2 + with: + python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + if: steps.defaults.outputs.proceed == 'true' + - name: > + Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch + so that the PR does not override it + # We should not override those scripts which become part of the image as they will not be + # changed in the image built - we should only override those that are executed to build + # the image. + run: | + rm -rf "scripts/ci" + mv "main-airflow/scripts/ci" "scripts" + if: steps.defaults.outputs.proceed == 'true' + - name: "Free space" + run: ./scripts/ci/tools/ci_free_space_on_ci.sh + if: steps.defaults.outputs.proceed == 'true' + - name: "Build CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh + # Pull images built in the previous step + if: steps.defaults.outputs.proceed == 'true' + env: + GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true" + - name: "Build PROD images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh + if: steps.defaults.outputs.proceed == 'true' + - name: "Push PROD images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + run: ./scripts/ci/images/ci_push_production_images.sh + if: steps.defaults.outputs.proceed == 'true' + + cancel-on-ci-build: + name: "Cancel 'CI Build' jobs on workflow failed/cancelled" + runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }} + if: failure() || cancelled() + needs: [build-ci-images, build-prod-images] + env: + branch: ${{ github.event.pull_request.head.ref }} + thisRun: ${{ github.run_id }} + steps: + - name: Find running CI Build jobs for ${{ env.TARGET_COMMIT_SHA }} + run: | + if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then + event_filter="event=pull_request&" + else + branch="${GITHUB_REF#refs/heads/}" + event_filter="" + fi + + for cancel_url in $( + gh api "/repos/$GITHUB_REPOSITORY/actions/runs?${event_filter}branch=${branch}" \ + jq -r ' + .workflow_runs[] | + select(.head_sha == $ENV.TARGET_COMMIT_SHA and .status != "completed") | + .cancel_url + ' \ + ); do + # One of these URls will be _this_ workflow, so lets exclude that! + [[ $cancel_url == */$thisRun/* ]] && continue + + echo "Cancelling $cancel_url" + gh api -X POST --silent "$cancel_url" + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 415b2d8b2e0447..2926173ee5d933 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ env: # In builds from forks, this token is empty, and this is good because such builds do not even try # to push images to the registry. CONTAINER_REGISTRY_TOKEN: ${{ secrets.PAT_CR }} - GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.run_id }}" + GITHUB_REGISTRY_PULL_IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" INSTALL_PROVIDERS_FROM_SOURCES: "true" @@ -68,6 +68,10 @@ env: # by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false" GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }} +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build-info: @@ -185,14 +189,6 @@ jobs: id: source-run-info with: token: ${{ secrets.GITHUB_TOKEN }} - - name: > - Event: ${{ github.event_name }} - Repo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} - Branch: ${{ github.head_ref }} - Run id: ${{ github.run_id }} - Sha: ${{ github.sha }} - Ref: ${{ github.ref }} - run: printenv - name: Set wait for image id: wait-for-image run: | @@ -413,6 +409,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}" env: VERBOSE: false + - run: sleep 600 static-checks-pylint: diff --git a/.gitmodules b/.gitmodules index 1779dddb765512..e03978e2636531 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule ".github/actions/get-workflow-origin"] path = .github/actions/get-workflow-origin url = https://github.com/potiuk/get-workflow-origin -[submodule ".github/actions/cancel-workflow-runs"] - path = .github/actions/cancel-workflow-runs - url = https://github.com/potiuk/cancel-workflow-runs [submodule ".github/actions/checks-action"] path = .github/actions/checks-action url = https://github.com/LouisBrunner/checks-action From f80b061c4e3f83bdcc1172b7e976aaa30fa91315 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor <83078142+AshsTestUser@users.noreply.github.com> Date: Fri, 23 Apr 2021 15:08:15 +0100 Subject: [PATCH 2/7] Test commit --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2209b04d4f0c8f..9045bdf79848f9 100644 --- a/README.md +++ b/README.md @@ -273,3 +273,5 @@ If you would love to have Apache Airflow stickers, t-shirt etc. then check out - [Documentation](https://airflow.apache.org/docs/apache-airflow/stable/) - [Chat](https://s.apache.org/airflow-slack) + +Fri 23 Apr 15:07:53 BST 2021 From 8f082e63eb03c0f1a9f2318e943fab7f263d2f77 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor <83078142+AshsTestUser@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:32:52 +0100 Subject: [PATCH 3/7] Second commit From 3464d68a93227a615798dc98c589417d5e26cb3c Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor <83078142+AshsTestUser@users.noreply.github.com> Date: Tue, 27 Apr 2021 14:11:29 +0100 Subject: [PATCH 4/7] Third commit From 3106c376c98810f8c92a5f47a54421db76e1fd3f Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor <83078142+AshsTestUser@users.noreply.github.com> Date: Tue, 27 Apr 2021 14:11:32 +0100 Subject: [PATCH 5/7] Forth commit From 1e30d21d135aaba143c0ec88e5cd0a71629a66fd Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor <83078142+AshsTestUser@users.noreply.github.com> Date: Wed, 19 May 2021 11:55:05 +0100 Subject: [PATCH 6/7] Fith commit From 6588af27d8e7d371c9dcd049bd0ee31e10992411 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor <83078142+AshsTestUser@users.noreply.github.com> Date: Wed, 19 May 2021 14:03:30 +0100 Subject: [PATCH 7/7] Sixth commit --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 89a077eb185e75..6b45977e7f8c53 100644 --- a/setup.py +++ b/setup.py @@ -1032,3 +1032,4 @@ def include_provider_namespace_packages_when_installing_from_sources() -> None: if __name__ == "__main__": do_setup() + print("Hi!")