Optimise caching of test results

.github/workflows/e2e.yml

@@ -1,3 +1,9 @@
+# We use github cache to save snapshots between runs.
+# For PRs and MergeQueues, the target commit is used, and for push events, github.event.previous is used.
+# If a snapshot for a given Hash is not found, we checkout that commit, run the tests and cache the snapshots.
+# These are then downloaded before running the E2E, providing the reference snapshots.
+# If there are any errors, the diff image is uploaded to artifacts, and the user is notified.
+
 name: E2E
 
 on:
@@ -8,10 +14,16 @@ on:
 permissions:
   contents: read
 
+env:
+  # For PRs and MergeQueues, the target commit is used, and for push events, github.event.previous is used.
+  targetHash: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.event.before }}
+
 jobs:
   cache:
     runs-on: ubuntu-latest
     steps:
+      - run: |
+          echo '${{ toJSON(github) }}' | sed 's/::/:/g'
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v2
       - name: Setup Node.js
@@ -25,13 +37,14 @@ jobs:
         with:
           save-always: true
           path: ./cypress/snapshots
-          key: ${{ runner.os }}-snapshots
+          key: ${{ runner.os }}-snapshots-${{ env.targetHash }}
 
+      # If a snapshot for a given Hash is not found, we checkout that commit, run the tests and cache the snapshots.
       - name: Switch to base branch
         if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }}
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base.sha || 'develop' }}
+          ref: ${{ env.targetHash }}
 
       - name: Cypress run
         uses: cypress-io/github-action@v4
@@ -61,12 +74,13 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
+      # These cached snapshots are downloaded, providing the reference snapshots.
       - name: Cache snapshots
         id: cache-snapshot
         uses: actions/cache/restore@v3
         with:
           path: ./cypress/snapshots
-          key: ${{ runner.os }}-snapshots
+          key: ${{ runner.os }}-snapshots-${{ env.targetHash }}
 
       # Install NPM dependencies, cache them correctly
       # and run all Cypress tests
@@ -101,9 +115,54 @@ jobs:
           verbose: true
           token: 6845cc80-77ee-4e17-85a1-026cd95e0766
 
+      # We upload the artifacts into numbered archives to prevent overwriting
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        if: ${{ failure() && steps.cypress.conclusion == 'failure' }}
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: snapshots-${{ matrix.containers }}
+          retention-days: 1
+          path: ./cypress/snapshots
+
+  combineArtifacts:
+    needs: e2e
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    steps:
+      # Download all snapshot artifacts and merge them into a single folder
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: snapshots
+          pattern: snapshots-*
+          merge-multiple: true
+
+      # For successful push events, we save the snapshots cache
+      - name: Save snapshots cache
+        id: cache-upload
+        if: ${{ github.event_name == 'push' && needs.e2e.result != 'failure' }}
+        uses: actions/cache/save@v3
+        with:
+          path: ./snapshots
+          key: ${{ runner.os }}-snapshots-${{ github.event.after }}
+
+      - name: Flatten images to a folder
+        if: ${{ needs.e2e.result == 'failure'  }}
+        run: |
+          mkdir errors
+          cd snapshots
+          find . -mindepth 2 -type d -name "*__diff_output__*" -exec sh -c 'mv "$0"/*.png ../errors/' {} \;
+
+      - name: Upload Error snapshots
+        if: ${{ needs.e2e.result == 'failure' }}
+        uses: actions/upload-artifact@v4
+        id: upload-artifacts
         with:
           name: error-snapshots
-          path: cypress/snapshots/**/__diff_output__/*
+          retention-days: 10
+          path: errors/
+
+      - name: Notify Users
+        if: ${{ needs.e2e.result == 'failure' }}
+        run: |
+          echo "::error title=Visual tests failed::You can view images that failed by downloading the error-snapshots artifact: ${{ steps.upload-artifacts.outputs.artifact-url }}"