refactor(ci): optimize CI/CD pipeline with parallel testing and enhan…

…ced caching

- Introduced a `Prepare` job to set up the repository, cache dependencies, count unit tests, and calculate chunks for parallel execution.
- Updated `Build-And-Test-Server` job to restore repository and dependencies from cache, run integration tests, and add results to the job summary.
- Added `Run-Unit-Tests` job to execute unit tests in parallel chunks, upload test results, and add them to the job summary.
- Enhanced `Build-And-Test-Web` job to cache Playwright browsers, run end-to-end tests, and upload E2E test artifacts.
- Refined `Release-on-NPM` and `Release-mina-signer-on-NPM` jobs to build and publish the project if the version has changed.

.github/workflows/build-action.yml

@@ -8,7 +8,75 @@ on:
   workflow_dispatch: {}
 
 jobs:
+  Prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      test_count: ${{ steps.count_tests.outputs.test_count }}
+      chunk_count: ${{ steps.calculate_chunks.outputs.chunk_count }}
+      chunk_array: ${{ steps.create_chunk_array.outputs.chunk_array }}
+    steps:
+      - name: Checkout repository with submodules
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+
+      - name: Cache dependencies and build
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: |
+            ~/.npm
+            node_modules
+            dist
+          key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }}
+
+      - name: Build o1js
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          npm ci
+          npm run build
+
+      - name: Count tests
+        id: count_tests
+        run: |
+          TEST_COUNT=$(find ./dist/node -name "*.unit-test.js" | wc -l)
+          echo "test_count=${TEST_COUNT}" >> $GITHUB_OUTPUT
+          echo "Total test count: ${TEST_COUNT}"
+
+      - name: Calculate number of chunks
+        id: calculate_chunks
+        run: |
+          test_count=${{ steps.count_tests.outputs.test_count }}
+          if [ $test_count -le 10 ]; then
+            echo "chunk_count=1" >> $GITHUB_OUTPUT
+          elif [ $test_count -le 20 ]; then
+            echo "chunk_count=2" >> $GITHUB_OUTPUT
+          elif [ $test_count -le 40 ]; then
+            echo "chunk_count=4" >> $GITHUB_OUTPUT
+          else
+            echo "chunk_count=8" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Create chunk array
+        id: create_chunk_array
+        run: |
+          chunk_count=${{ steps.calculate_chunks.outputs.chunk_count }}
+          chunk_array=$(seq -s ',' 1 $chunk_count)
+          echo "chunk_array=[${chunk_array}]" >> $GITHUB_OUTPUT
+
+      - name: Cache repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
   Build-And-Test-Server:
+    needs: Prepare
     timeout-minutes: 210
     runs-on: ubuntu-latest
     strategy:
@@ -21,51 +89,160 @@ jobs:
             'DEX integration tests',
             'DEX integration test with proofs',
             'Voting integration tests',
-            'Unit tests',
             'Verification Key Regression Check',
             'CommonJS test',
           ]
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
-      - name: Build o1js and execute tests
+
+      - name: Restore cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.npm
+            node_modules
+            dist
+          key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }}
+
+      - name: Prepare for tests
+        run: touch profiling.md
+
+      - name: Execute tests
         env:
           TEST_TYPE: ${{ matrix.test_type }}
+        run: sh run-ci-tests.sh
+
+      - name: Add to job summary
+        if: always()
+        run: |
+          echo "### Test Results for ${{ matrix.test_type }}" >> $GITHUB_STEP_SUMMARY
+          cat profiling.md >> $GITHUB_STEP_SUMMARY
+
+  Run-Unit-Tests:
+    needs: Prepare
+    name: Run unit tests parallel
+    timeout-minutes: 210
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        chunk: ${{ fromJson(needs.Prepare.outputs.chunk_array) }}
+    steps:
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+
+      - name: Restore cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.npm
+            node_modules
+            dist
+          key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }}
+
+      - name: Prepare for tests
+        run: touch profiling.md
+
+      - name: Run unit tests
+        env:
+          TOTAL_TESTS: ${{ needs.Prepare.outputs.test_count }}
+          CHUNK: ${{ matrix.chunk }}
+          CHUNKS: ${{ needs.Prepare.outputs.chunk_count }}
+        run: |
+          echo "Total tests: $TOTAL_TESTS"
+          echo "Current chunk: $CHUNK"
+          echo "Total chunks: $CHUNKS"
+
+          if [ -z "$TOTAL_TESTS" ] || [ "$TOTAL_TESTS" -eq 0 ]; then
+            echo "Error: TOTAL_TESTS is not set or is zero. Exiting."
+            exit 1
+          fi
+
+          start_index=$(( (TOTAL_TESTS * (CHUNK - 1) / CHUNKS) ))
+          end_index=$(( (TOTAL_TESTS * CHUNK / CHUNKS) ))
+
+          echo "Running tests from index $start_index to $end_index"
+
+          shopt -s globstar
+          test_files=(./dist/node/**/*.unit-test.js)
+
+          for ((i=start_index; i<end_index && i<${#test_files[@]}; i++)); do
+            echo "Running test: ${test_files[$i]}"
+            node --enable-source-maps "${test_files[$i]}" | tee -a profiling.md
+          done
         continue-on-error: false
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ matrix.chunk }}
+          path: profiling.md
+
+      - name: Add to job summary
+        if: always()
         run: |
-          git submodule update --init --recursive
-          npm ci
-          npm run build
-          touch profiling.md
-          sh run-ci-tests.sh
+          echo "### Test Results for Unit Tests Chunk ${{ matrix.chunk }}" >> $GITHUB_STEP_SUMMARY
           cat profiling.md >> $GITHUB_STEP_SUMMARY
 
   Build-And-Test-Web:
+    needs: Prepare
     timeout-minutes: 90
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
-      - name: Install Node dependencies
-        run: |
-          git submodule update --init --recursive
-          npm ci
+
+      - name: Restore npm cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}
+
+      - name: Cache Playwright browsers
+        uses: actions/cache@v4
+        id: playwright-cache
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.OS }}-playwright-${{ hashFiles('**/package-lock.json') }}
+
       - name: Install Playwright browsers
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
         run: npm run e2e:install
+
       - name: Build o1js and prepare the web server
         run: |
           npm run build:web
           npm run e2e:prepare-server
+
       - name: Execute E2E tests
         run: npm run test:e2e
+
       - name: Upload E2E test artifacts
         uses: actions/upload-artifact@v4
         continue-on-error: true
@@ -80,19 +257,24 @@ jobs:
     if: github.ref == 'refs/heads/main'
     timeout-minutes: 180
     runs-on: ubuntu-latest
-    needs: [Build-And-Test-Server, Build-And-Test-Web]
+    needs: [Build-And-Test-Server, Run-Unit-Tests, Build-And-Test-Web]
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
+
       - name: Build o1js
         run: |
-          git submodule update --init --recursive
           npm ci
           npm run prepublishOnly
+
       - name: Publish to NPM if version has changed
         uses: JS-DevTools/npm-publish@v3
         with:
@@ -105,21 +287,26 @@ jobs:
     if: github.ref == 'refs/heads/main'
     timeout-minutes: 180
     runs-on: ubuntu-latest
-    needs: [Build-And-Test-Server, Build-And-Test-Web]
+    needs: [Build-And-Test-Server, Run-Unit-Tests, Build-And-Test-Web]
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Restore repository
+        uses: actions/cache@v4
+        with:
+          path: .
+          key: repo-${{ github.sha }}
+
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
           node-version: '18'
+
       - name: Build mina-signer
         run: |
-          git submodule update --init --recursive
           npm ci
           cd src/mina-signer
           npm ci
           npm run prepublishOnly
+
       - name: Publish to NPM if version has changed
         uses: JS-DevTools/npm-publish@v3
         with: