From 5559adf15c322fb216c05caed529858614d683ae Mon Sep 17 00:00:00 2001
From: Anton Belodedenko <2033996+ab77@users.noreply.github.com>
Date: Mon, 22 Apr 2024 09:43:07 -0700
Subject: [PATCH] update CI config

---
 .github/workflows/balena.yml | 886 +++++++++++++++++------------------
 1 file changed, 434 insertions(+), 452 deletions(-)

diff --git a/.github/workflows/balena.yml b/.github/workflows/balena.yml
index 18378d496..1480776b3 100644
--- a/.github/workflows/balena.yml
+++ b/.github/workflows/balena.yml
@@ -4,9 +4,7 @@ name: deploy openBalena
 on:
   pull_request:
     types: [opened, synchronize, closed]
-    branches:
-      - master
-      - main
+    branches: [master, main]
 
 env:
   # nested virtualisation not supported on AWS/EC2 instance types|classes other than X.metal
@@ -42,24 +40,23 @@ jobs:
     timeout-minutes: 120
     strategy:
       fail-fast: true
-
     steps:
-      - uses: actions/checkout@v2
-        with:
-          # FIXME: remove once balenaBlocks/balenaVirt is a thing
-          submodules: true
+    - uses: actions/checkout@v2
+      with:
+        # FIXME: remove once balenaBlocks/balenaVirt is a thing
+        submodules: true
 
-    - uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # v1.8.0
+    - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
       id: ephemeral
       with:
         app_id: ${{ vars.APP_ID || '291899' }}
-        installation_id: ${{ vars.INSTALLATION_ID || '34046749' }}
+        installation_retrieval_mode: id
+        installation_retrieval_payload: ${{ vars.INSTALLATION_ID || '34046749' }}
         permissions: >-
           {
             "actions": "read",
             "contents": "read",
             "metadata": "read",
-            "packages": "read",
             "pull_requests": "write"
           }
         private_key: ${{ secrets.GH_APP_PRIVATE_KEY }}
@@ -100,571 +97,556 @@ jobs:
           || echo "${RUNNER_TEMP}/ssh-uuid" >> "${GITHUB_PATH}"
 
 
-      - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
-        id: ephemeral
-        with:
-          app_id: ${{ vars.APP_ID || '291899' }}
-          installation_retrieval_mode: id
-          installation_retrieval_payload: ${{ vars.INSTALLATION_ID || '34046749' }}
-          permissions: >-
-            {
-              "actions": "read",
-              "contents": "read",
-              "metadata": "read",
-              "pull_requests": "write"
-            }
-          private_key: ${{ secrets.GH_APP_PRIVATE_KEY }}
-
-      - name: push draft or finalise release
-        timeout-minutes: 60
-        id: push-release
-        uses: balena-io/deploy-to-balena-action@master
-        with:
-          balena_token: ${{ secrets.BALENA_API_KEY }}
-          cache: false
-          environment: ${{ env.ENVIRONMENT }}
-          fleet: ${{ env.FLEET }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          # FIXME: remove once balenaBlocks/balenaVirt is a thing
-          versionbot: false
-          registry_secrets: |
-            {
-              "ghcr.io": {
-                "username": "${{ github.actor }}",
-                "password": "${{ steps.ephemeral.outputs.token }}"
-              },
-              "docker.io": {
-                "username": "${{ secrets.DOCKERHUB_USER }}",
-                "password": "${{ secrets.DOCKERHUB_TOKEN }}"
-              }
+    - name: push draft or finalise release
+      timeout-minutes: 60
+      id: push-release
+      uses: balena-io/deploy-to-balena-action@master
+      with:
+        balena_token: ${{ secrets.BALENA_API_KEY }}
+        cache: false
+        environment: ${{ env.ENVIRONMENT }}
+        fleet: ${{ env.FLEET }}
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        # FIXME: remove once balenaBlocks/balenaVirt is a thing
+        versionbot: false
+        registry_secrets: |
+          {
+            "ghcr.io": {
+              "username": "${{ github.actor }}",
+              "password": "${{ steps.ephemeral.outputs.token }}"
+            },
+            "docker.io": {
+              "username": "${{ secrets.DOCKERHUB_USER }}",
+              "password": "${{ secrets.DOCKERHUB_TOKEN }}"
             }
+          }
 
-      - name: (pre)register test device
-        id: register-test-device
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        run: |
-          set -ue
+    - name: (pre)register test device
+      id: register-test-device
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-          balena_device_uuid="$(openssl rand -hex 16)"
+        balena_device_uuid="$(openssl rand -hex 16)"
 
-          # https://www.balena.io/docs/learn/more/masterclasses/advanced-cli/#52-preregistering-a-device
-          balena device register '${{ env.FLEET }}' --uuid "${balena_device_uuid}"
+        # https://www.balena.io/docs/learn/more/masterclasses/advanced-cli/#52-preregistering-a-device
+        balena device register '${{ env.FLEET }}' --uuid "${balena_device_uuid}"
 
-          device_id="$(balena device "${balena_device_uuid}" | grep ^ID: | cut -c20-)"
+        device_id="$(balena device "${balena_device_uuid}" | grep ^ID: | cut -c20-)"
 
-          # the actual version deployed depends on the AWS EC2/AMI, defined in AWS_EC2_LAUNCH_TEMPLATE
-          os_version="$(balena os versions ${{ env.DEVICE_TYPE }} \
-            | grep '${{ env.VARIANT }}' | head -n 1 | sed 's/.${{ env.VARIANT }}//g')"
+        # the actual version deployed depends on the AWS EC2/AMI, defined in AWS_EC2_LAUNCH_TEMPLATE
+        os_version="$(balena os versions ${{ env.DEVICE_TYPE }} \
+          | grep '${{ env.VARIANT }}' | head -n 1 | sed 's/.${{ env.VARIANT }}//g')"
 
-          balena config generate \
-            --version "${os_version}" \
-            --device ${balena_device_uuid} \
-            --network ethernet \
-            --appUpdatePollInterval 10 \
-            --output config.json
+        balena config generate \
+          --version "${os_version}" \
+          --device "${balena_device_uuid}" \
+          --network ethernet \
+          --appUpdatePollInterval 10 \
+          --output config.json
 
-          balena tag set balena ephemeral-test-device --device "${balena_device_uuid}"
+        balena tag set balena ephemeral-test-device --device "${balena_device_uuid}"
 
-          github_vars=(GITHUB_ACTOR GITHUB_BASE_REF GITHUB_HEAD_REF GITHUB_JOB \
-            GITHUB_REF GITHUB_REF_NAME GITHUB_REF_TYPE GITHUB_REPOSITORY \
-            GITHUB_REPOSITORY_OWNER GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_RUN_NUMBER \
-            GITHUB_SHA GITHUB_WORKFLOW RUNNER_ARCH RUNNER_NAME RUNNER_OS)
+        github_vars=(GITHUB_ACTOR GITHUB_BASE_REF GITHUB_HEAD_REF GITHUB_JOB \
+          GITHUB_REF GITHUB_REF_NAME GITHUB_REF_TYPE GITHUB_REPOSITORY \
+          GITHUB_REPOSITORY_OWNER GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_RUN_NUMBER \
+          GITHUB_SHA GITHUB_WORKFLOW RUNNER_ARCH RUNNER_NAME RUNNER_OS)
 
-          for github_var in "${github_vars[@]}"; do
-              balena tag set ${github_var} "${!github_var}" --device "${balena_device_uuid}"
-          done
+        for github_var in "${github_vars[@]}"; do
+            balena tag set ${github_var} "${!github_var}" --device "${balena_device_uuid}"
+        done
 
-          echo "balena_device_uuid=${balena_device_uuid}" >> "${GITHUB_OUTPUT}"
-          echo "balena_device_id=${device_id}" >> "${GITHUB_OUTPUT}"
+        echo "balena_device_uuid=${balena_device_uuid}" >> "${GITHUB_OUTPUT}"
+        echo "balena_device_id=${device_id}" >> "${GITHUB_OUTPUT}"
 
-     # https://github.com/balena-io/balena-cli/issues/1543
-      - name: pin device to draft release
-        id: pin-device
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        run: |
-          set -uae
+   # https://github.com/balena-io/balena-cli/issues/1543
+    - name: pin device to draft release
+      id: pin-device
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -uae
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-          balena_releases="$(mktemp)"
-          balena releases '${{ env.FLEET }}' | tail -n +2 | head -n ${{ env.RELEASES }} > "${balena_releases}"
+        balena_releases="$(mktemp)"
+        balena releases '${{ env.FLEET }}' | tail -n +2 | head -n ${{ env.RELEASES }} > "${balena_releases}"
 
-          # convert to JSON to find the correct draft release id and commit
-          release_id="$(while IFS=' ' read -r id commit created_at status semver is_final
-          do
-            printf '{"id":%s,"commit":"%s","created_at":"%s","status":"%s","semver":"%s","is_final":%s}\n' \
-              "${id}" "${commit}" "${created_at}" "${status}" "${semver}" "${is_final}"
-          done < "${balena_releases}" | jq -s | jq -r '.[] | select((.id==${{ steps.push-release.outputs.release_id }}) and (.is_final==false) and (.status=="success")).id')"
+        # convert to JSON to find the correct draft release id and commit
+        release_id="$(while IFS=' ' read -r id commit created_at status semver is_final
+        do
+          printf '{"id":%s,"commit":"%s","created_at":"%s","status":"%s","semver":"%s","is_final":%s}\n' \
+            "${id}" "${commit}" "${created_at}" "${status}" "${semver}" "${is_final}"
+        done < "${balena_releases}" | jq -s | jq -r '.[] | select((.id==${{ steps.push-release.outputs.release_id }}) and (.is_final==false) and (.status=="success")).id')"
 
-          commit="$(while IFS=' ' read -r id commit created_at status semver is_final
-          do
-            printf '{"id":%s,"commit":"%s","created_at":"%s","status":"%s","semver":"%s","is_final":%s}\n' \
-              "${id}" "${commit}" "${created_at}" "${status}" "${semver}" "${is_final}"
-          done < "${balena_releases}" | jq -s | jq -r '.[] | select(.id==${{ steps.push-release.outputs.release_id }}).commit')"
+        commit="$(while IFS=' ' read -r id commit created_at status semver is_final
+        do
+          printf '{"id":%s,"commit":"%s","created_at":"%s","status":"%s","semver":"%s","is_final":%s}\n' \
+            "${id}" "${commit}" "${created_at}" "${status}" "${semver}" "${is_final}"
+        done < "${balena_releases}" | jq -s | jq -r '.[] | select(.id==${{ steps.push-release.outputs.release_id }}).commit')"
 
-          if ! [ '${{ steps.register-test-device.outputs.balena_device_id }}' = '' ] \
-            && ! [ "${release_id}" = '' ] \
-            && ! [ "${commit}" = '' ]; then
-              # pin DUT to draft release
-              curl -X PATCH --silent --retry ${{ env.RETRY }} --fail -o /dev/null \
-                'https://api.${{ env.ENVIRONMENT }}/v6/device?$filter=id%20in%20(${{ steps.register-test-device.outputs.balena_device_id }})' \
-                -H 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
-                -H 'content-type: application/json' \
-                --data-raw "{\"should_be_running__release\":${release_id}}" \
-                --compressed
-          fi
+        if ! [ '${{ steps.register-test-device.outputs.balena_device_id }}' = '' ] \
+          && ! [ "${release_id}" = '' ] \
+          && ! [ "${commit}" = '' ]; then
+            # pin DUT to draft release
+            curl -X PATCH --silent --retry ${{ env.RETRY }} --fail -o /dev/null \
+              'https://api.${{ env.ENVIRONMENT }}/v6/device?$filter=id%20in%20(${{ steps.register-test-device.outputs.balena_device_id }})' \
+              -H 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
+              -H 'content-type: application/json' \
+              --data-raw "{\"should_be_running__release\":${release_id}}" \
+              --compressed
+        fi
 
-          balena device ${{ steps.register-test-device.outputs.balena_device_uuid }}
+        balena device ${{ steps.register-test-device.outputs.balena_device_uuid }}
 
-          app_id="$(balena fleet ${{ env.FLEET }} | grep ^ID: | cut -c14-)"
+        app_id="$(balena fleet ${{ env.FLEET }} | grep ^ID: | cut -c14-)"
 
-          echo "balena_app_id=${app_id}" >> "${GITHUB_OUTPUT}"
+        echo "balena_app_id=${app_id}" >> "${GITHUB_OUTPUT}"
 
-      - name: configure test device environment
-        id: configure-test-env
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        run: |
-          set -ue
+    - name: configure test device environment
+      id: configure-test-env
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-          balena env add VERBOSE '${{ env.VERBOSE }}' \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add VERBOSE '${{ env.VERBOSE }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add DNS_TLD '${{ env.DNS_TLD }}' \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add DNS_TLD '${{ env.DNS_TLD }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add MDNS_TLD '' \
-            --service mdns \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add MDNS_TLD '' \
+          --service mdns \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add DB_HOST db \
-            --service api \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add DB_HOST db \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add REDIS_HOST redis:6379 \
-            --service api \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add REDIS_HOST redis:6379 \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
 
-          # to allow devices running locally to communicate to the local API, we can route
-          # to the local Docker network aliases instead of public DNS, since (a) DNS_TLD is
-          # guestfwd(ed) in QEMU to a special internal IP 10.0.2.100; (b) is proxied to
-          # haproxy network alias on device; and (c) made public with a wildcard DNS record
-          # (e.g.)
-          #
-          # $ dig +short api.auto.balena-devices.com
-          # 10.0.2.100
-          #
-
-          balena env add API_HOST 'api.${{ env.DNS_TLD }}' \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
-
-          # not used but required for config.json to be valid
-          balena env add DELTA_HOST 'delta.${{ env.DNS_TLD }}' \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        # to allow devices running locally to communicate to the local API, we can route
+        # to the local Docker network aliases instead of public DNS, since (a) DNS_TLD is
+        # guestfwd(ed) in QEMU to a special internal IP 10.0.2.100; (b) is proxied to
+        # haproxy network alias on device; and (c) made public with a wildcard DNS record
+        # (e.g.)
+        #
+        # $ dig +short api.auto.balena-devices.com
+        # 10.0.2.100
+        #
 
-          balena env add REGISTRY2_HOST 'registry.${{ env.DNS_TLD }}' \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add API_HOST 'api.${{ env.DNS_TLD }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add VPN_HOST 'vpn.${{ env.DNS_TLD }}' \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        # not used but required for config.json to be valid
+        balena env add DELTA_HOST 'delta.${{ env.DNS_TLD }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add HOST 'api.${{ env.DNS_TLD }}' \
-            --service api \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add REGISTRY2_HOST 'registry.${{ env.DNS_TLD }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add TOKEN_AUTH_CERT_ISSUER 'api.${{ env.DNS_TLD }}' \
-            --service api \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add VPN_HOST 'vpn.${{ env.DNS_TLD }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add REGISTRY2_TOKEN_AUTH_ISSUER 'api.${{ env.DNS_TLD }}' \
-            --service registry \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add HOST 'api.${{ env.DNS_TLD }}' \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add REGISTRY2_TOKEN_AUTH_REALM 'https://api.${{ env.DNS_TLD }}/auth/v1/token' \
-            --service registry \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add TOKEN_AUTH_CERT_ISSUER 'api.${{ env.DNS_TLD }}' \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add REGISTRY2_S3_REGION_ENDPOINT 's3.${{ env.DNS_TLD }}' \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
-
-          balena env add PRODUCTION_MODE false \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add REGISTRY2_TOKEN_AUTH_ISSUER 'api.${{ env.DNS_TLD }}' \
+          --service registry \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena env add COMMON_REGION '${{ env.AWS_REGION }}' \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
-
-          balena env add SUPERUSER_EMAIL 'admin@${{ env.DNS_TLD }}' \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
-
-          balena env add ORG_UNIT openBalena \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
-
-      - name: configure test device secrets
-        id: configure-test-secrets
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        run: |
-          set -ue
+        balena env add REGISTRY2_TOKEN_AUTH_REALM 'https://api.${{ env.DNS_TLD }}/auth/v1/token' \
+          --service registry \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        balena env add REGISTRY2_S3_REGION_ENDPOINT 's3.${{ env.DNS_TLD }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        balena env add PRODUCTION_MODE false \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          # cert-manager requires it to get whoami information for the user
-          balena env add API_TOKEN '${{ secrets.BALENA_API_KEY }}' \
-            --service cert-manager \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add COMMON_REGION '${{ env.AWS_REGION }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          # cert-manager requires is to request wildcard SSL certificate from LetsEncrypt
-          balena env add CLOUDFLARE_API_TOKEN '${{ secrets.CLOUDFLARE_API_TOKEN }}' \
-            --service cert-manager \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add SUPERUSER_EMAIL 'admin@${{ env.DNS_TLD }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          # AWS credentials to backup/restore PKI assets
-          with_backoff balena env add AWS_ACCESS_KEY_ID '${{ env.AWS_ACCESS_KEY_ID }}' \
-            --service cert-manager \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+        balena env add ORG_UNIT openBalena \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          with_backoff balena env add AWS_SECRET_ACCESS_KEY '${{ env.AWS_SECRET_ACCESS_KEY }}' \
-            --service cert-manager \
-            --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+    - name: configure test device secrets
+      id: configure-test-secrets
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
 
-      - name: provision ephemeral test device
-        id: provision-test-device
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        run: |
-          set -ue
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-          for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
-              for market_type in ${{ env.MARKET_TYPES }}; do
-                  # https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
-                  response="$(aws ec2 run-instances \
-                    --launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \
-                    --instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \
-                    $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
-                    --security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \
-                    --subnet-id "${subnet_id}" \
-                    --associate-public-ip-address \
-                    --user-data file://config.json \
-                    --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)"
+        # cert-manager requires it to get whoami information for the user
+        balena env add API_TOKEN '${{ secrets.BALENA_API_KEY }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-                  [[ -n $response ]] && break
-              done
-              [[ -n $response ]] && break
-          done
+        # cert-manager requires is to request wildcard SSL certificate from LetsEncrypt
+        balena env add CLOUDFLARE_API_TOKEN '${{ secrets.CLOUDFLARE_API_TOKEN }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          [[ -z $response ]] && exit 1
+        # AWS credentials to backup/restore PKI assets
+        with_backoff balena env add AWS_ACCESS_KEY_ID '${{ env.AWS_ACCESS_KEY_ID }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          instance_id="$(echo "${response}" | jq -r '.Instances[].InstanceId')"
+        with_backoff balena env add AWS_SECRET_ACCESS_KEY '${{ env.AWS_SECRET_ACCESS_KEY }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
 
-          aws ec2 wait instance-running --instance-ids "${instance_id}"
+    - name: provision ephemeral test device
+      id: provision-test-device
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
 
-          aws ec2 wait instance-status-ok --instance-ids "${instance_id}"
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          echo "instance_id=${instance_id}" >> "${GITHUB_OUTPUT}"
+        for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
+            for market_type in ${{ env.MARKET_TYPES }}; do
+                # https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
+                response="$(aws ec2 run-instances \
+                  --launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \
+                  --instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \
+                  $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
+                  --security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \
+                  --subnet-id "${subnet_id}" \
+                  --associate-public-ip-address \
+                  --user-data file://config.json \
+                  --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)"
 
-        env:
-          AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
+                [[ -n $response ]] && break
+            done
+            [[ -n $response ]] && break
+        done
 
-      - name: provision SSH key
-        id: provision-ssh-key
-        # wait for cloud-config
-        # https://github.com/balena-os/cloud-config
-        timeout-minutes: 10
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        run: |
-          set -ue
+        [[ -z $response ]] && exit 1
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        instance_id="$(echo "${response}" | jq -r '.Instances[].InstanceId')"
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        aws ec2 wait instance-running --instance-ids "${instance_id}"
 
-          if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then
-              ssh-keygen -N '' \
-                -C "$(balena whoami | grep EMAIL | cut -c11-)" \
-                -f "${HOME}/.ssh/id_rsa"
-          fi
+        aws ec2 wait instance-status-ok --instance-ids "${instance_id}"
 
-          match=''
-          for key in $(balena keys | grep -v ID | awk '{print $1}'); do
-              fp=$(balena key ${key} | tail -n 1 | ssh-keygen -E md5 -lf /dev/stdin | awk '{print $2}')
-              if [[ $fp =~ $(ssh-keygen -E md5 -lf "${HOME}/.ssh/id_rsa" | awk '{print $2}') ]]; then
-                  match="${key}"
-                  break
-              fi
-          done
-
-          if [[ -z $match ]]; then
-              balena key add "${GITHUB_SHA}" "${HOME}/.ssh/id_rsa.pub"
-          else
-              balena keys
-          fi
-
-          pgrep ssh-agent || ssh-agent -a "${SSH_AUTH_SOCK}"
+        echo "instance_id=${instance_id}" >> "${GITHUB_OUTPUT}"
 
-          ssh-add "${HOME}/.ssh/id_rsa"
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
 
-          while ! [[ "$(ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
-            cat /mnt/boot/config.json | jq -r .uuid)" =~ ${{ steps.register-test-device.outputs.balena_device_uuid }} ]]; do
+    - name: provision SSH key
+      id: provision-ssh-key
+      # wait for cloud-config
+      # https://github.com/balena-os/cloud-config
+      timeout-minutes: 10
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
 
-              echo "::warning::Still working..."
-              sleep "$(( (RANDOM % 5) + 5 ))s"
-          done
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-        env:
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then
+            ssh-keygen -N '' \
+              -C "$(balena whoami | grep EMAIL | cut -c11-)" \
+              -f "${HOME}/.ssh/id_rsa"
+        fi
 
-      # fall-back to QEMU software emulation when nested virtualisation is not available
-      - name: create dummy kvm device
-        id: nested-virtualisation-bypass
-        timeout-minutes: 10
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        run: |
-          set -ue
+        match=''
+        for key in $(balena keys | grep -v ID | awk '{print $1}'); do
+            fp=$(balena key ${key} | tail -n 1 | ssh-keygen -E md5 -lf /dev/stdin | awk '{print $2}')
+            if [[ $fp =~ $(ssh-keygen -E md5 -lf "${HOME}/.ssh/id_rsa" | awk '{print $2}') ]]; then
+                match="${key}"
+                break
+            fi
+        done
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        if [[ -z $match ]]; then
+            balena key add "${GITHUB_SHA}" "${HOME}/.ssh/id_rsa.pub"
+        else
+            balena keys
+        fi
 
-          source functions
+        pgrep ssh-agent || ssh-agent -a "${SSH_AUTH_SOCK}"
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        ssh-add "${HOME}/.ssh/id_rsa"
 
-          balena whoami && ssh-add -l
+        while ! [[ "$(ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          cat /mnt/boot/config.json | jq -r .uuid)" =~ ${{ steps.register-test-device.outputs.balena_device_uuid }} ]]; do
 
-          with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
-            '[[ -e /dev/kvm ]] || mknod /dev/kvm b 1 1'
-
-        env:
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
 
-      - name: wait for application
-        id: wait-application
-        timeout-minutes: 30
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        run: |
-          set -ue
+        echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
 
-          source functions
+    # fall-back to QEMU software emulation when nested virtualisation is not available
+    - name: create dummy kvm device
+      id: nested-virtualisation-bypass
+      timeout-minutes: 10
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          balena whoami && ssh-add -l
+        source functions
 
-          while [[ "$(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
-            'https://api.${{ env.ENVIRONMENT }}/supervisor/v1/device' \
-            --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
-            --header 'Content-Type:application/json' \
-            --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
-            --compressed | jq -r '.update_pending')" =~ ^true$ ]]; do
-
-              sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
-          done
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-          # wait for services to start running
-          while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
-            'balena ps -q | xargs balena inspect | jq -r .[].State.Status' \
-            | grep -E 'created|restarting|removing|paused|exited|dead'; do
+        balena whoami && ssh-add -l
 
-              echo "::warning::Still working..."
-              sleep "$(( (RANDOM % 30) + 30 ))s"
-          done
+        with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          '[[ -e /dev/kvm ]] || mknod /dev/kvm b 1 1'
 
-          # wait for Docker healthchecks
-          while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
-            'balena ps -q | xargs balena inspect \
-            | jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
-            | grep -E ':starting|:unhealthy'; do
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
 
-              echo "::warning::Still working..."
-              sleep "$(( (RANDOM % 30) + 30 ))s"
-          done
+    - name: wait for application
+      id: wait-application
+      timeout-minutes: 30
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
 
-        env:
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-      # (TBC) https://www.balena.io/docs/reference/supervisor/docker-compose/
-      # due to lack of long form depends_on support in compositions, restart to ensure all
-      # components are running with the latest configuration; preferred over restart via
-      # Supervisor API restart due to potential HTTP [timeouts](https://github.com/balena-os/balena-supervisor/issues/1157)
-      - name: restart components
-        id: restart-application
-        timeout-minutes: 15
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        run: |
-          set -ue
+        source functions
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-          source functions
+        balena whoami && ssh-add -l
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        while [[ "$(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
+          'https://api.${{ env.ENVIRONMENT }}/supervisor/v1/device' \
+          --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
+          --header 'Content-Type:application/json' \
+          --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
+          --compressed | jq -r '.update_pending')" =~ ^true$ ]]; do
 
-          balena whoami && ssh-add -l
+            sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
+        done
 
-          with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
-            'balena ps -q | xargs balena restart || true'
+        # wait for services to start running
+        while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena inspect | jq -r .[].State.Status' \
+          | grep -E 'created|restarting|removing|paused|exited|dead'; do
 
-          #balena device restart ${{ steps.register-test-device.outputs.balena_device_uuid }}
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 30) + 30 ))s"
+        done
 
-          # wait for Docker healthchecks
-          while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
-            'balena ps -q | xargs balena inspect \
-            | jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
-            | grep -vE 'resin_supervisor|balena_supervisor' \
-            | grep -E ':starting|:unhealthy'; do
+        # wait for Docker healthchecks
+        while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena inspect \
+          | jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
+          | grep -E ':starting|:unhealthy'; do
 
-              echo "::warning::Still working..."
-              sleep "$(( (RANDOM % 30) + 30 ))s"
-          done
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 30) + 30 ))s"
+        done
 
-        env:
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
 
-      - name: device tests
-        id: device-tests
-        if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
-        # https://giters.com/gfx/example-github-actions-with-tty
-        # https://github.com/actions/runner/issues/241#issuecomment-924327172
-        shell: 'script -q -e -c "bash {0}"'
-        run: |
-          set -ue
+    # (TBC) https://www.balena.io/docs/reference/supervisor/docker-compose/
+    # due to lack of long form depends_on support in compositions, restart to ensure all
+    # components are running with the latest configuration; preferred over restart via
+    # Supervisor API restart due to potential HTTP [timeouts](https://github.com/balena-os/balena-supervisor/issues/1157)
+    - name: restart components
+      id: restart-application
+      timeout-minutes: 15
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          source functions
+        source functions
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-          balena whoami && ssh-add -l
+        balena whoami && ssh-add -l
 
-          while ! [[ $(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
-            'https://api.${{ env.ENVIRONMENT }}/supervisor/v2/applications/state' \
-            --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
-            --header 'Content-Type:application/json' \
-            --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
-            --compressed | jq -r '.[].services."${{ env.OPENBALENA_TESTS_SERVICE }}".status') =~ Run|run ]]; do
+        with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena restart || true'
 
-              echo "::warning::Still working..."
-              sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
-          done
+        #balena device restart ${{ steps.register-test-device.outputs.balena_device_uuid }}
 
-          # (TBC) placeholder for a complete end-to-end test suite
-          with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-            --service ${{ env.OPENBALENA_TESTS_SERVICE }} \
-            ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
-            '/root/run-tests.sh'
+        # wait for Docker healthchecks
+        while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena inspect \
+          | jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
+          | grep -vE 'resin_supervisor|balena_supervisor' \
+          | grep -E ':starting|:unhealthy'; do
 
-        env:
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-          ATTEMPTS: 2
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 30) + 30 ))s"
+        done
 
-      - name: remove SSH key
-        if: always()
-        id: remove-ssh-key
-        run: |
-          set -ue
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+    - name: device tests
+      id: device-tests
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      # https://giters.com/gfx/example-github-actions-with-tty
+      # https://github.com/actions/runner/issues/241#issuecomment-924327172
+      shell: 'script -q -e -c "bash {0}"'
+      run: |
+        set -ue
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          balena keys | grep ${{ steps.provision-ssh-key.outputs.key_id }} \
-            | awk '{print $1}' | xargs balena key rm --yes || true
+        source functions
 
-          pgrep ssh-agent && (pgrep ssh-agent | xargs kill)
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-          rm -f /tmp/ssh_agent.sock
+        balena whoami && ssh-add -l
 
-      - name: destroy ephemeral test device
-        if: always()
-        id: destroy-test-device
-        run: |
-          set -ue
+        while ! [[ $(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
+          'https://api.${{ env.ENVIRONMENT }}/supervisor/v2/applications/state' \
+          --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
+          --header 'Content-Type:application/json' \
+          --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
+          --compressed | jq -r '.[].services."${{ env.OPENBALENA_TESTS_SERVICE }}".status') =~ Run|run ]]; do
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+            echo "::warning::Still working..."
+            sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
+        done
 
-          balena login --token '${{ secrets.BALENA_API_KEY }}'
+        # (TBC) placeholder for a complete end-to-end test suite
+        with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          --service ${{ env.OPENBALENA_TESTS_SERVICE }} \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          '/root/run-tests.sh'
 
-          aws ec2 terminate-instances \
-            --instance-ids ${{ steps.provision-test-device.outputs.instance_id }} || true
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        ATTEMPTS: 2
 
-          balena device rm ${{ steps.register-test-device.outputs.balena_device_uuid }} --yes || true
+    - name: remove SSH key
+      if: always()
+      id: remove-ssh-key
+      run: |
+        set -ue
 
-        env:
-          AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-      # remove orphaned ACME DNS-01 validation records
-      # https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
-      # FIXME: clean up older _acme-challenge.auto TXT records
-      - name: cleanup-dns-records
-        if: always()
-        id: cloudflare-dns-cleanup
-        run: |
-          set -ue
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        balena keys | grep ${{ steps.provision-ssh-key.outputs.key_id }} \
+          | awk '{print $1}' | xargs balena key rm --yes || true
 
-          subdomain="$(echo "${DNS_TLD}" | sed "s/.${DEVICE_PROXY_TLD}//g")"
-          match="${{ steps.register-test-device.outputs.balena_device_uuid }}.${subdomain}"
+        pgrep ssh-agent && (pgrep ssh-agent | xargs kill)
 
-          zone_id="$(curl --silent --retry ${{ env.RETRY }} \
-            "https://api.cloudflare.com/client/v4/zones?name=${DEVICE_PROXY_TLD}" \
-            -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' | jq -r '.result[].id')"
+        rm -f /tmp/ssh_agent.sock
 
-          for record in "$(curl --silent --retry ${{ env.RETRY }} \
-            "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
-            -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' \
-            | jq -r --arg match "${match}" '.result[] | select(((.type=="TXT") and (.name | contains($match))))' \
-            | base64)"; do
+    - name: destroy ephemeral test device
+      if: always()
+      id: destroy-test-device
+      run: |
+        set -ue
+
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        balena login --token '${{ secrets.BALENA_API_KEY }}'
 
-              json="$(echo "${record}" | base64 -d | jq -r)"
-              id="$(echo "${json}" | jq -r .id)"
-              name="$(echo "${json}" | jq -r .name)"
+        aws ec2 terminate-instances \
+          --instance-ids ${{ steps.provision-test-device.outputs.instance_id }} || true
 
-              if [[ -n $id ]] && [[ -n $name ]]; then
-                  echo "::warning::Orphaned DNS record ${name} (${id})..."
+        balena device rm ${{ steps.register-test-device.outputs.balena_device_uuid }} --yes || true
 
-                  if [[ -z $DRY_RUN ]]; then
-                      curl -X DELETE --silent --retry ${{ env.RETRY }} \
-                        "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${id}" \
-                        -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}'
-                  fi
-              fi
-          done
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
 
-        env:
-          DRY_RUN: true
+    # remove orphaned ACME DNS-01 validation records
+    # https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
+    # FIXME: clean up older _acme-challenge.auto TXT records
+    - name: cleanup-dns-records
+      if: always()
+      id: cloudflare-dns-cleanup
+      run: |
+        set -ue
 
-      - name: remove registry secrets
-        if: always()
-        id: remove-registry-secrets
-        run: |
-          set -ue
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        subdomain="$(echo "${DNS_TLD}" | sed "s/.${DEVICE_PROXY_TLD}//g")"
+        match="${{ steps.register-test-device.outputs.balena_device_uuid }}.${subdomain}"
+
+        zone_id="$(curl --silent --retry ${{ env.RETRY }} \
+          "https://api.cloudflare.com/client/v4/zones?name=${DEVICE_PROXY_TLD}" \
+          -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' | jq -r '.result[].id')"
+
+        for record in "$(curl --silent --retry ${{ env.RETRY }} \
+          "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
+          -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' \
+          | jq -r --arg match "${match}" '.result[] | select(((.type=="TXT") and (.name | contains($match))))' \
+          | base64)"; do
+
+            json="$(echo "${record}" | base64 -d | jq -r)"
+            id="$(echo "${json}" | jq -r .id)"
+            name="$(echo "${json}" | jq -r .name)"
+
+            if [[ -n $id ]] && [[ -n $name ]]; then
+                echo "::warning::Orphaned DNS record ${name} (${id})..."
+
+                if [[ -z $DRY_RUN ]]; then
+                    curl -X DELETE --silent --retry ${{ env.RETRY }} \
+                      "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${id}" \
+                      -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}'
+                fi
+            fi
+        done
+
+      env:
+        DRY_RUN: true
+
+    - name: remove registry secrets
+      if: always()
+      id: remove-registry-secrets
+      run: |
+        set -ue
 
-          [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+        [[ '${{ env.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
 
-          rm -f "${HOME}/.balena/secrets.json"
+        rm -f "${HOME}/.balena/secrets.json"