diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml index 9726c5acad86a..c5480c6fbcfc4 100644 --- a/.github/workflows/build-images-workflow-run.yml +++ b/.github/workflows/build-images-workflow-run.yml @@ -30,7 +30,6 @@ env: SKIP_CHECK_REMOTE_IMAGE: "true" DB_RESET: "true" VERBOSE: "true" - UPGRADE_TO_LATEST_CONSTRAINTS: false USE_GITHUB_REGISTRY: "true" GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_USERNAME: ${{ github.actor }} @@ -57,7 +56,6 @@ jobs: sourceEvent: ${{ steps.source-run-info.outputs.sourceEvent }} cacheDirective: ${{ steps.cache-directive.outputs.docker-cache }} buildImages: ${{ steps.build-images.outputs.buildImages }} - upgradeToLatestConstraints: ${{ steps.upgrade-constraints.outputs.upgradeToLatestConstraints }} steps: - name: "Get information about the original trigger of the run" uses: potiuk/get-workflow-origin@588cc14f9f1cdf1b8be3db816855e96422204fec # v1_3 @@ -153,15 +151,6 @@ jobs: else echo "::set-output name=docker-cache::pulled" fi - - name: "Set upgrade to latest constraints" - id: upgrade-constraints - run: | - if [[ ${{ steps.cancel.outputs.sourceEvent == 'push' || - steps.cancel.outputs.sourceEvent == 'scheduled' }} == 'true' ]]; then - echo "::set-output name=upgradeToLatestConstraints::${{ github.sha }}" - else - echo "::set-output name=upgradeToLatestConstraints::false" - fi - name: "Cancel all duplicated 'Build Image' runs" # We find duplicates of all "Build Image" runs - due to a missing feature # in GitHub Actions, we have to use Job names to match Event/Repo/Branch matching @@ -198,6 +187,7 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} outputs: pythonVersions: ${{ steps.selective-checks.python-versions }} + upgradeToLatestConstraints: ${{ steps.selective-checks.outputs.upgrade-to-latest-constraints }} allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} @@ -243,12 +233,12 @@ jobs: id: selective-checks env: EVENT_NAME: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }} - INCOMING_COMMIT_SHA: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} + TARGET_COMMIT_SHA: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} PR_LABELS: ${{ needs.cancel-workflow-runs.outputs.pullRequestLabels }} run: | if [[ ${EVENT_NAME} == "pull_request" ]]; then # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}" + ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" else # Run all checks ./scripts/ci/selective_ci_checks.sh @@ -273,7 +263,7 @@ jobs: BACKEND: postgres PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.workflow_run.id }} - UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.cancel-workflow-runs.outputs.upgradeToLatestConstraints }} + UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }} DOCKER_CACHE: ${{ needs.cancel-workflow-runs.outputs.cacheDirective }} steps: - name: > diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5931135df50cb..77cbf65b5ed05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,6 @@ env: SKIP_CHECK_REMOTE_IMAGE: "true" DB_RESET: "true" VERBOSE: "true" - UPGRADE_TO_LATEST_CONSTRAINTS: ${{ github.event_name == 'push' || github.event_name == 'scheduled' }} DOCKER_CACHE: "pulled" USE_GITHUB_REGISTRY: "true" GITHUB_REPOSITORY: ${{ github.repository }} @@ -69,6 +68,7 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} outputs: waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }} + upgradeToLatestConstraints: ${{ steps.selective-checks.outputs.upgrade-to-latest-constraints }} pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} @@ -131,12 +131,12 @@ jobs: id: selective-checks env: EVENT_NAME: ${{ github.event_name }} - INCOMING_COMMIT_SHA: ${{ github.sha }} + TARGET_COMMIT_SHA: ${{ github.sha }} PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" run: | if [[ ${EVENT_NAME} == "pull_request" ]]; then # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}" + ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" else # Run all checks ./scripts/ci/selective_ci_checks.sh @@ -150,6 +150,7 @@ jobs: if: needs.build-info.outputs.image-build == 'true' env: BACKEND: sqlite + UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -568,7 +569,8 @@ jobs: needs: [build-info] env: BACKEND: sqlite - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + UPGRADE_TO_LATEST_CONSTRAINTS: ${{ needs.build-info.outputs.upgradeToLatestConstraints }} if: needs.build-info.outputs.image-build == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2bf92b788afbd..9fa7b94594ce8 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -40,11 +40,11 @@ jobs: id: selective-checks env: EVENT_NAME: ${{ github.event_name }} - INCOMING_COMMIT_SHA: ${{ github.sha }} + TARGET_COMMIT_SHA: ${{ github.sha }} run: | if [[ ${EVENT_NAME} == "pull_request" ]]; then # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}" + ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" else # Run all checks ./scripts/ci/selective_ci_checks.sh diff --git a/.github/workflows/label_when_reviewed_workflow_run.yml b/.github/workflows/label_when_reviewed_workflow_run.yml index 6e45038395609..6ea15b04aa7b4 100644 --- a/.github/workflows/label_when_reviewed_workflow_run.yml +++ b/.github/workflows/label_when_reviewed_workflow_run.yml @@ -75,12 +75,12 @@ jobs: id: selective-checks env: EVENT_NAME: ${{ steps.source-run-info.outputs.sourceEvent }} - INCOMING_COMMIT_SHA: ${{ steps.source-run-info.outputs.targetCommitSha }} + TARGET_COMMIT_SHA: ${{ steps.source-run-info.outputs.targetCommitSha }} PR_LABELS: ${{ steps.source-run-info.outputs.pullRequestLabels }} run: | if [[ ${EVENT_NAME} == "pull_request_review" ]]; then # Run selective checks - ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}" + ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" else # Run all checks ./scripts/ci/selective_ci_checks.sh diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6d34026932ed0..61883e7ab7096 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -321,6 +321,11 @@ Step 4: Prepare PR the "full tests needed" label is set for your PR. Additional check is set that prevents from accidental merging of the request until full matrix of tests succeeds for the PR. + * when your change has "upgrade to latest dependencies" label set, constraints will be automatically + upgraded to latest constraints matching your setup.py. This is useful in case you want to force + upgrade to a latest version of dependencies. You can ask committers to set the label for you + when you need it in your PR. + More details about the PR workflow be found in `PULL_REQUEST_WORKFLOW.rst `_. diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index 3c7132d44245c..a6c66ebf8a397 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -34,16 +34,39 @@ if [[ ${PR_LABELS=} == *"full tests needed"* ]]; then echo echo "Found the right PR labels in '${PR_LABELS=}': 'full tests needed''" echo - FULL_TESTS_NEEDED="true" + FULL_TESTS_NEEDED_LABEL="true" else echo echo "Did not find the right PR labels in '${PR_LABELS=}': 'full tests needed'" echo - FULL_TESTS_NEEDED="false" + FULL_TESTS_NEEDED_LABEL="false" +fi + +if [[ ${PR_LABELS=} == *"upgrade to latest dependencies"* ]]; then + echo + echo "Found the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies''" + echo + UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="true" +else + echo + echo "Did not find the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies'" + echo + UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="false" fi function output_all_basic_variables() { - if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then + if [[ "${UPGRADE_TO_LATEST_CONSTRAINTS_LABEL}" == "true" || + ${EVENT_NAME} == 'push' || ${EVENT_NAME} == "scheduled" ]]; then + # Trigger upgrading to latest constraints where label is set or when + # SHA of the merge commit triggers rebuilding layer in the docker image + # Each build that upgrades to latest constraints will get truly latest constraints, not those + # Cached in the image this way + initialization::ga_output upgrade-to-latest-constraints "${INCOMING_COMMIT_SHA}" + else + initialization::ga_output upgrade-to-latest-constraints "false" + fi + + if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then initialization::ga_output python-versions \ "$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" initialization::ga_output all-python-versions \ @@ -60,7 +83,7 @@ function output_all_basic_variables() { fi initialization::ga_output default-python-version "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" - if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then + if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then initialization::ga_output kubernetes-versions \ "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_VERSIONS[@]}")" else @@ -73,7 +96,7 @@ function output_all_basic_variables() { "$(initialization::parameters_to_json "${CURRENT_KUBERNETES_MODES[@]}")" initialization::ga_output default-kubernetes-mode "${KUBERNETES_MODE}" - if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then + if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then initialization::ga_output postgres-versions \ "$(initialization::parameters_to_json "${CURRENT_POSTGRES_VERSIONS[@]}")" else @@ -82,7 +105,7 @@ function output_all_basic_variables() { fi initialization::ga_output default-postgres-version "${POSTGRES_VERSION}" - if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then + if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then initialization::ga_output mysql-versions \ "$(initialization::parameters_to_json "${CURRENT_MYSQL_VERSIONS[@]}")" else @@ -100,7 +123,7 @@ function output_all_basic_variables() { "$(initialization::parameters_to_json "${CURRENT_HELM_VERSIONS[@]}")" initialization::ga_output default-helm-version "${HELM_VERSION}" - if [[ ${FULL_TESTS_NEEDED} == "true" ]]; then + if [[ ${FULL_TESTS_NEEDED_LABEL} == "true" ]]; then initialization::ga_output postgres-exclude '[{ "python-version": "3.6" }]' initialization::ga_output mysql-exclude '[{ "python-version": "3.7" }]' initialization::ga_output sqlite-exclude '[{ "python-version": "3.8" }]' @@ -114,9 +137,6 @@ function output_all_basic_variables() { } function get_changed_files() { - INCOMING_COMMIT_SHA="${1}" - readonly INCOMING_COMMIT_SHA - echo echo "Incoming commit SHA: ${INCOMING_COMMIT_SHA}" echo @@ -414,14 +434,20 @@ if (($# < 1)); then echo echo "No Commit SHA - running all tests (likely direct master merge, or scheduled run)!" echo - # override FULL_TESTS_NEEDED in master/scheduled run - FULL_TESTS_NEEDED="true" - readonly FULL_TESTS_NEEDED + INCOMING_COMMIT_SHA="" + readonly INCOMING_COMMIT_SHA + # override FULL_TESTS_NEEDED_LABEL in master/scheduled run + FULL_TESTS_NEEDED_LABEL="true" + readonly FULL_TESTS_NEEDED_LABEL output_all_basic_variables set_outputs_run_everything_and_exit +else + INCOMING_COMMIT_SHA="${1}" + readonly INCOMING_COMMIT_SHA fi -readonly FULL_TESTS_NEEDED + +readonly FULL_TESTS_NEEDED_LABEL output_all_basic_variables image_build_needed="false" @@ -429,7 +455,7 @@ docs_build_needed="false" tests_needed="false" kubernetes_tests_needed="false" -get_changed_files "${1}" +get_changed_files run_all_tests_if_environment_files_changed check_if_docs_should_be_generated check_if_helm_tests_should_be_run