Implement release locking to avoid publishing helm chart before conta…

…iner image (#3940)

* added lock around release PR as guardrail to avoid out of ordering artefacts publishing

* update pr.yaml to just be triggered when PRs

* support failure handling while updating release status check

* added comments and error propagation for release locking

.github/workflows/pr.yaml

@@ -267,6 +267,23 @@ jobs:
     steps:
       - run: echo "All done"
 
+  #  release step updates 'release' status check for non releases branches. See ../../doc/incidents/issues-3907 for full context.
+  release:
+    if: ${{ github.event_name == 'pull_request' && !startsWith(github.event.pull_request.head.ref, 'releases/') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Release
+        run: |
+          curl --fail --request POST \
+            --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
+            --header 'authorization: Bearer ${{ secrets.WEAVE_GITOPS_BOT_ACCESS_TOKEN }}' \
+            --header 'content-type: application/json' \
+            --data '{
+                "state":"success",
+                "description":"release not required",
+                "context":"release"
+              }'
+
   notify-failure:
     name: Notify Slack on Failure
     runs-on: ubuntu-latest

.github/workflows/prepare-release.yaml

@@ -115,6 +115,18 @@ jobs:
            ${{ steps.github_release.outputs.changelog }}
           token: ${{ secrets.WEAVE_GITOPS_BOT_ACCESS_TOKEN }}
           labels: "exclude from release notes"
+      #  'Lock Release PR Merge' sets 'release' status check with pending state to avoid accidentally merging the release PR. See ../../doc/incidents/issues-3907 for full context.
+      - name: Lock Release PR
+        run: |
+          curl --fail --request POST \
+            --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ steps.create-pull-request.outputs.pull-request-head-sha }} \
+            --header 'authorization: Bearer ${{ secrets.WEAVE_GITOPS_BOT_ACCESS_TOKEN }}' \
+            --header 'content-type: application/json' \
+            --data '{
+                "state":"pending",
+                "description":"execute the release to pass this check",
+                "context":"release"
+              }'
       - name: "Comment on pull request"
         run: |
           curl --request POST \

.github/workflows/release.yaml

@@ -168,6 +168,18 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+      #  'Unlock Release PR Merge' sets 'release' status check state to success to unlock merging the release PR. See ../../doc/incidents/issues-3907 for full context.
+      - name: Unlock Release PR Merge
+        run: |
+          curl --fail --request POST \
+            --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
+            --header 'authorization: Bearer ${{ secrets.WEAVE_GITOPS_BOT_ACCESS_TOKEN }}' \
+            --header 'content-type: application/json' \
+            --data '{
+                "state":"success",
+                "description":"release happened. PR ready to merge",
+                "context":"release"
+              }'
       - name: "Merge release"
         run: |
           curl --request PUT \

doc/release-process.md

@@ -48,9 +48,7 @@ To release a new version of Weave Gitops, you need to:
     called "Doc site preview")
 - The PR cover message contains draft release notes. Edit the cover
   message to fill in or delete blocks as appropriate. In case of making a fix release because of regressions, add a message about the regression to the release notes. Move as many PRs out of "Uncategorized" as you possibly can.
-- If everything looks good, approve the PR - do *not* merge or things
-  won't be published in the right order. This immediately kicks off the
-  release job.
+- If everything looks good, approve the PR to trigger the [release](../.github/workflows/release.yaml) workflow.
 - Wait for the action to finish, at which point the PR will be merged automatically.
 - Notify weave-gitops-dev channel that PRs are now safe to merge.