Refactor GitHub Actions Workflows for Improved CI/CD Performance

.github/workflows/build-action.yml

@@ -8,7 +8,53 @@ on:
   workflow_dispatch: {}
 
 jobs:
+  Prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      test_count: ${{ steps.count_tests.outputs.test_count }}
+      chunk_count: 8 # This is hardcoded to 8, but it can be changed to any number.
+    steps:
+      - name: Checkout repository with submodules
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+
+      - name: Cache dependencies and build
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: |
+            ~/.npm
+            node_modules
+            dist
+          key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }}
+
+      - name: Build o1js
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          npm ci
+          npm run build
+
+      - name: Count tests
+        id: count_tests
+        run: |
+          TEST_COUNT=$(find ./dist/node -name "*.unit-test.js" | wc -l)
+          echo "test_count=${TEST_COUNT}" >> $GITHUB_OUTPUT
+          echo "Total test count: ${TEST_COUNT}"
+
+      - name: Cache repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
   Build-And-Test-Server:
+    needs: Prepare
     timeout-minutes: 210
     runs-on: ubuntu-latest
     strategy:
@@ -21,51 +67,167 @@ jobs:
             'DEX integration tests',
             'DEX integration test with proofs',
             'Voting integration tests',
-            'Unit tests',
             'Verification Key Regression Check',
             'CommonJS test',
           ]
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
-      - name: Build o1js and execute tests
+
+      - name: Restore cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.npm
+            node_modules
+            dist
+          key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }}
+
+      - name: Prepare for tests
+        run: touch profiling.md
+
+      - name: Execute tests
         env:
           TEST_TYPE: ${{ matrix.test_type }}
+        run: sh run-ci-tests.sh
+
+      - name: Add to job summary
+        if: always()
+        run: |
+          echo "### Test Results for ${{ matrix.test_type }}" >> $GITHUB_STEP_SUMMARY
+          cat profiling.md >> $GITHUB_STEP_SUMMARY
+
+  Run-Unit-Tests:
+    needs: Prepare
+    name: Run unit tests parallel
+    timeout-minutes: 210
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        chunk: [1, 2, 3, 4, 5, 6, 7, 8]
+    steps:
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+
+      - name: Restore cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.npm
+            node_modules
+            dist
+          key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }}
+
+      - name: Prepare for tests
+        run: touch profiling.md
+
+      - name: Run unit tests
+        env:
+          TOTAL_TESTS: ${{ needs.Prepare.outputs.test_count }}
+          CHUNK: ${{ matrix.chunk }}
+          CHUNKS: 8
+        run: |
+          echo "Total tests: $TOTAL_TESTS"
+          echo "Current chunk: $CHUNK"
+          echo "Total chunks: $CHUNKS"
+
+          if [ -z "$TOTAL_TESTS" ] || [ "$TOTAL_TESTS" -eq 0 ]; then
+            echo "Error: TOTAL_TESTS is not set or is zero. Exiting."
+            exit 1
+          fi
+
+          start_index=$(( (TOTAL_TESTS * (CHUNK - 1) / CHUNKS) ))
+          end_index=$(( (TOTAL_TESTS * CHUNK / CHUNKS) ))
+
+          echo "Running tests from index $start_index to $end_index"
+
+          shopt -s globstar
+          test_files=(./dist/node/**/*.unit-test.js)
+
+          for ((i=start_index; i<end_index && i<${#test_files[@]}; i++)); do
+            echo "Running test: ${test_files[$i]}"
+            node --enable-source-maps "${test_files[$i]}" | tee -a profiling.md
+          done
         continue-on-error: false
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ matrix.chunk }}
+          path: profiling.md
+
+      - name: Add to job summary
+        if: always()
         run: |
-          git submodule update --init --recursive
-          npm ci
-          npm run build
-          touch profiling.md
-          sh run-ci-tests.sh
+          echo "### Test Results for Unit Tests Chunk ${{ matrix.chunk }}" >> $GITHUB_STEP_SUMMARY
           cat profiling.md >> $GITHUB_STEP_SUMMARY
 
+  Build-And-Test-Server-Unit-Tests:
+    name: Build-And-Test-Server (Unit tests)
+    needs: [Run-Unit-Tests]
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "All unit tests completed successfully"
+
   Build-And-Test-Web:
+    needs: Prepare
     timeout-minutes: 90
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
-      - name: Install Node dependencies
-        run: |
-          git submodule update --init --recursive
-          npm ci
+
+      - name: Restore npm cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}
+
+      - name: Cache Playwright browsers
+        uses: actions/cache@v4
+        id: playwright-cache
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.OS }}-playwright-${{ hashFiles('**/package-lock.json') }}
+
       - name: Install Playwright browsers
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
         run: npm run e2e:install
+
       - name: Build o1js and prepare the web server
         run: |
           npm run build:web
           npm run e2e:prepare-server
+
       - name: Execute E2E tests
         run: npm run test:e2e
+
       - name: Upload E2E test artifacts
         uses: actions/upload-artifact@v4
         continue-on-error: true
@@ -80,19 +242,24 @@ jobs:
     if: github.ref == 'refs/heads/main'
     timeout-minutes: 180
     runs-on: ubuntu-latest
-    needs: [Build-And-Test-Server, Build-And-Test-Web]
+    needs: [Build-And-Test-Server, Run-Unit-Tests, Build-And-Test-Web]
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
+
       - name: Build o1js
         run: |
-          git submodule update --init --recursive
           npm ci
           npm run prepublishOnly
+
       - name: Publish to NPM if version has changed
         uses: JS-DevTools/npm-publish@v3
         with:
@@ -105,21 +272,26 @@ jobs:
     if: github.ref == 'refs/heads/main'
     timeout-minutes: 180
     runs-on: ubuntu-latest
-    needs: [Build-And-Test-Server, Build-And-Test-Web]
+    needs: [Build-And-Test-Server, Run-Unit-Tests, Build-And-Test-Web]
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
+
       - name: Build mina-signer
         run: |
-          git submodule update --init --recursive
           npm ci
           cd src/mina-signer
           npm ci
           npm run prepublishOnly
+
       - name: Publish to NPM if version has changed
         uses: JS-DevTools/npm-publish@v3
         with:

run-ci-tests.sh

@@ -34,15 +34,6 @@ case $TEST_TYPE in
   ./run src/examples/zkapps/dex/happy-path-with-proofs.ts --bundle
   ;;
 
-"Unit tests")
-  echo "Running unit tests"
-  cd src/mina-signer
-  npm run build
-  cd ../..
-  npm run test:unit
-  npm run test
-  ;;
-
 "Verification Key Regression Check")
   echo "Running Regression checks"
   ./run ./tests/vk-regression/vk-regression.ts --bundle