feat: simplify production release reverts (#678)

Update the Production Terraform workflows so that a release can
more easily be reverted. Now all that will be required to rollback a
failed release is to revert the Release Please PR.

.github/workflows/get-version/action.yml

@@ -0,0 +1,40 @@
+name: Get infrastructure version to deploy
+
+inputs:
+  is-tagged:
+    description: "Is this for the release of a tagged version?  This action will wait for the tag to exist if so."
+    required: true
+outputs:
+  version:
+    description: "Infrastructure version to deploy"
+    value: ${{ steps.output-version.outputs.version }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Get version, release PR branch
+      if: inputs.is-tagged == 'false' && startsWith(github.head_ref, 'release-please--')
+      env:
+        GITHUB_HEAD_REF: ${{ github.head_ref }}
+      run: echo "version=$GITHUB_HEAD_REF" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Get version, perform release of tag
+      if: inputs.is-tagged == 'true'
+      run: echo "version=v$(cat version.txt)" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Wait for tag to exist
+      if: inputs.is-tagged == 'true'
+      run: ./.github/workflows/scripts/wait-for-tag.sh ${{ env.version }}
+      shell: bash
+
+    - name: Fail if no version set
+      if: env.version == ''   
+      run: exit 1
+      shell: bash
+
+    - name: Output version
+      id: output-version
+      run: echo "version=${{ env.version }}" >> $GITHUB_OUTPUT
+      shell: bash

.github/workflows/scripts/wait-for-tag.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+#
+# Given a git tag name as an argument, this script will wait until the tag is available in the remote repository.
+# The wait is based on the values of CHECK_INTERVAL and MAX_CHECKS.
+# 
+# Usage:
+#   ./wait-for-tag.sh v1.2.3
+#
+
+set -euo pipefail
+IFS=$'\n\t'
+
+TAG_NAME="$1"
+CHECK_INTERVAL=5
+MAX_CHECKS=20
+
+for ((i=1; i<=MAX_CHECKS; i++)); do
+  git fetch --tags > /dev/null 2>&1
+  if git rev-parse "$TAG_NAME" >/dev/null 2>&1; then
+    echo "🎉 Tag $TAG_NAME exists!"
+    exit 0
+  fi
+  echo "Tag $TAG_NAME not found. Checking again in $CHECK_INTERVAL seconds... (Attempt $i/$MAX_CHECKS)"
+  sleep $CHECK_INTERVAL
+done
+
+echo "💀 Tag $TAG_NAME does not exist after $MAX_CHECKS attempts..."
+exit 1

.github/workflows/terragrunt-apply-production.yml

@@ -1,8 +1,11 @@
 name: "Terragrunt apply PRODUCTION"
 
 on:
-  release:
-    types: [published]
+  push:
+    branches:
+      - "develop"
+    paths:
+      - "version.txt"
 
 permissions:
   id-token: write
@@ -32,12 +35,31 @@ env:
   TF_VAR_email_address_support: ${{ vars.PRODUCTION_SUPPORT_EMAIL }}
 
 jobs:
+  get-version:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get-version.outputs.version }}    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Get version to deploy
+        id: get-version
+        uses: ./.github/workflows/get-version
+        with:
+          is-tagged: 'true'
+
   # We deploy ECR first to make sure it is available for the 'build-tag-push-lambda-images' job which will be run in parallel with `terragrunt-apply-all-modules`
   terragrunt-apply-ecr-only:
+    needs: get-version
     runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.get-version.outputs.version }}    
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ env.VERSION }}
 
       # Setup Terraform, Terragrunt, and Conftest
       - name: Setup terraform tools
@@ -55,15 +77,19 @@ jobs:
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
 
   build-tag-push-lambda-images:
-    needs: terragrunt-apply-ecr-only
+    needs: [get-version, terragrunt-apply-ecr-only]
     runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.get-version.outputs.version }}
     strategy:
       fail-fast: false
       matrix:
         image: [audit-logs, audit-logs-archiver, cognito-email-sender, cognito-pre-sign-up, form-archiver, nagware, notify-slack, reliability, reliability-dlq-consumer, response-archiver, submission, vault-integrity]
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ env.VERSION }}
 
       - name: Build Lambda images
         uses: ./.github/workflows/build-lambda-images
@@ -78,15 +104,19 @@ jobs:
           aws-role-session-name: TFApply
           aws-region: ${{ env.AWS_REGION }}
           lambda-name: ${{ matrix.image }}
-          image-tag: ${{ github.ref_name }}
+          image-tag: ${{ env.VERSION }}
 
   terragrunt-apply-all-modules:
-    needs: build-tag-push-lambda-images
+    needs: [get-version, build-tag-push-lambda-images]
     if: ${{ !failure() && !cancelled() }}
     runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.get-version.outputs.version }}
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ env.VERSION }}
 
       # Setup Terraform, Terragrunt, and Conftest
       - name: Setup terraform tools
@@ -173,28 +203,28 @@ jobs:
         run: terragrunt apply --terragrunt-non-interactive -auto-approve
 
   update-lambda-function-image:
-    needs: terragrunt-apply-all-modules
+    needs: [get-version, terragrunt-apply-all-modules]
     runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.get-version.outputs.version }}
     strategy:
       fail-fast: false
       matrix:
         image: [audit-logs, audit-logs-archiver, cognito-email-sender, cognito-pre-sign-up, form-archiver, nagware, notify-slack, reliability, reliability-dlq-consumer, response-archiver, submission, vault-integrity]
     steps:
-      - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
       - name: Request Lambda functions to use new image
         uses: ./.github/workflows/request-lambda-functions-to-use-new-image
         with:
           aws-role-to-assume: arn:aws:iam::${{ env.AWS_ACCOUNT_ID }}:role/forms-terraform-apply-release
           aws-role-session-name: TFApply
           aws-region: ${{ env.AWS_REGION }}
           lambda-name: ${{ matrix.image }}
-          image-tag: ${{ github.ref_name }}
+          image-tag: ${{ env.VERSION }}
 
   notify-on-error:
     needs:
       [
+        get-version,
         terragrunt-apply-ecr-only,
         build-tag-push-lambda-images,
         terragrunt-apply-all-modules,

.github/workflows/terragrunt-apply-staging.yml

@@ -210,9 +210,6 @@ jobs:
       matrix:
         image: ${{ fromJSON(needs.detect-lambda-changes.outputs.lambda-to-rebuild) }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
       - name: Request Lambda functions to use new image
         uses: ./.github/workflows/request-lambda-functions-to-use-new-image
         with:

.github/workflows/terragrunt-plan-production.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     branches:
       - "develop"
+    paths:
+      - "version.txt"   
 
 permissions:
   id-token: write
@@ -35,14 +37,32 @@ env:
   TF_VAR_email_address_support: ${{ vars.PRODUCTION_SUPPORT_EMAIL }}
 
 jobs:
+  get-version:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get-version.outputs.version }}    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Get version to deploy
+        id: get-version
+        uses: ./.github/workflows/get-version
+        with:
+          is-tagged: ${{ !startsWith(github.head_ref, 'release-please--') }} # If not the release PR branch, assume it is a revert PR
+
   detect-lambda-changes:
-    if: startsWith(github.head_ref, 'release-please--')
+    needs: get-version
     runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.get-version.outputs.version }}  
     outputs:
       lambda-to-rebuild: ${{ steps.filter.outputs.changes }}
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ env.VERSION }}
 
       - name: Filter
         id: filter
@@ -51,16 +71,20 @@ jobs:
           filters: .github/lambda-filter.yml
 
   test-lambda-code:
-    needs: detect-lambda-changes
+    needs: [get-version, detect-lambda-changes]
     if: needs.detect-lambda-changes.outputs.lambda-to-rebuild != '[]'
     runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.get-version.outputs.version }}
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJSON(needs.detect-lambda-changes.outputs.lambda-to-rebuild) }}
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ env.VERSION }}
 
       - name: Test Lambda code
         uses: ./.github/workflows/test-lambda-code
@@ -69,16 +93,20 @@ jobs:
           lambda-name: ${{ matrix.image }}
 
   build-lambda-images:
-    needs: detect-lambda-changes
+    needs: [get-version, detect-lambda-changes]
     if: needs.detect-lambda-changes.outputs.lambda-to-rebuild != '[]'
     runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.get-version.outputs.version }}
     strategy:
       fail-fast: false
       matrix:
         image: ${{ fromJSON(needs.detect-lambda-changes.outputs.lambda-to-rebuild) }}
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ env.VERSION }}
 
       - name: Build Lambda images
         uses: ./.github/workflows/build-lambda-images
@@ -87,11 +115,15 @@ jobs:
           lambda-name: ${{ matrix.image }}
 
   terragrunt-plan:
-    if: startsWith(github.head_ref, 'release-please--')
+    needs: get-version
     runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.get-version.outputs.version }}
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ env.VERSION }}
 
       # Setup Terraform, Terragrunt, and Conftest
       - name: Setup terraform tools