diff --git a/.github/actions/live-tests-shared/action.yml b/.github/actions/live-tests-shared/action.yml index 15576fbfe5..6dda1b2f85 100644 --- a/.github/actions/live-tests-shared/action.yml +++ b/.github/actions/live-tests-shared/action.yml @@ -21,7 +21,7 @@ runs: env: USE_CUSTOM_LOCAL_NETWORK: 'true' run: | - git submodule update --init --recursive + GIT_LFS_SKIP_SMUDGE=1 git submodule update --init --recursive npm ci npm run build touch profiling.md diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index b234c94a07..ec1a4d969d 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -33,7 +33,7 @@ jobs: METRICS_SOURCE_ENVIRONMENT: 'o1js GitHub Actions' METRICS_BASE_BRANCH_FOR_COMPARISON: 'main' run: | - git submodule update --init --recursive + GIT_LFS_SKIP_SMUDGE=1 git submodule update --init --recursive npm ci npm run build echo 'Running o1js benchmarks.' diff --git a/.github/workflows/build-action.yml b/.github/workflows/build-action.yml index b01291da79..88898761b8 100644 --- a/.github/workflows/build-action.yml +++ b/.github/workflows/build-action.yml @@ -8,7 +8,53 @@ on: workflow_dispatch: {} jobs: + Prepare: + runs-on: ubuntu-latest + outputs: + test_count: ${{ steps.count_tests.outputs.test_count }} + chunk_count: 8 # This is hardcoded to 8, but it can be changed to any number. + steps: + - name: Checkout repository with submodules + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Cache dependencies and build + uses: actions/cache@v4 + id: cache + with: + path: | + ~/.npm + node_modules + dist + key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }} + + - name: Build o1js + if: steps.cache.outputs.cache-hit != 'true' + run: | + npm ci + npm run build + + - name: Count tests + id: count_tests + run: | + TEST_COUNT=$(find ./dist/node -name "*.unit-test.js" | wc -l) + echo "test_count=${TEST_COUNT}" >> $GITHUB_OUTPUT + echo "Total test count: ${TEST_COUNT}" + + - name: Cache repository + uses: actions/cache@v4 + with: + path: . + key: repo-${{ github.sha }} + Build-And-Test-Server: + needs: Prepare timeout-minutes: 210 runs-on: ubuntu-latest strategy: @@ -21,51 +67,167 @@ jobs: 'DEX integration tests', 'DEX integration test with proofs', 'Voting integration tests', - 'Unit tests', 'Verification Key Regression Check', 'CommonJS test', ] steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Restore repository + uses: actions/cache@v4 + with: + path: . + key: repo-${{ github.sha }} + - name: Setup Node uses: actions/setup-node@v4 with: node-version: '18' - - name: Build o1js and execute tests + + - name: Restore cache + uses: actions/cache@v4 + with: + path: | + ~/.npm + node_modules + dist + key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }} + + - name: Prepare for tests + run: touch profiling.md + + - name: Execute tests env: TEST_TYPE: ${{ matrix.test_type }} + run: sh run-ci-tests.sh + + - name: Add to job summary + if: always() + run: | + echo "### Test Results for ${{ matrix.test_type }}" >> $GITHUB_STEP_SUMMARY + cat profiling.md >> $GITHUB_STEP_SUMMARY + + Run-Unit-Tests: + needs: Prepare + name: Run unit tests parallel + timeout-minutes: 210 + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + chunk: [1, 2, 3, 4, 5, 6, 7, 8] + steps: + - name: Restore repository + uses: actions/cache@v4 + with: + path: . + key: repo-${{ github.sha }} + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Restore cache + uses: actions/cache@v4 + with: + path: | + ~/.npm + node_modules + dist + key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }} + + - name: Prepare for tests + run: touch profiling.md + + - name: Run unit tests + env: + TOTAL_TESTS: ${{ needs.Prepare.outputs.test_count }} + CHUNK: ${{ matrix.chunk }} + CHUNKS: 8 + run: | + echo "Total tests: $TOTAL_TESTS" + echo "Current chunk: $CHUNK" + echo "Total chunks: $CHUNKS" + + if [ -z "$TOTAL_TESTS" ] || [ "$TOTAL_TESTS" -eq 0 ]; then + echo "Error: TOTAL_TESTS is not set or is zero. Exiting." + exit 1 + fi + + start_index=$(( (TOTAL_TESTS * (CHUNK - 1) / CHUNKS) )) + end_index=$(( (TOTAL_TESTS * CHUNK / CHUNKS) )) + + echo "Running tests from index $start_index to $end_index" + + shopt -s globstar + test_files=(./dist/node/**/*.unit-test.js) + + for ((i=start_index; i> $GITHUB_STEP_SUMMARY cat profiling.md >> $GITHUB_STEP_SUMMARY + Build-And-Test-Server-Unit-Tests: + name: Build-And-Test-Server (Unit tests) + needs: [Run-Unit-Tests] + runs-on: ubuntu-latest + steps: + - run: echo "All unit tests completed successfully" + Build-And-Test-Web: + needs: Prepare timeout-minutes: 90 runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Restore repository + uses: actions/cache@v4 + with: + path: . + key: repo-${{ github.sha }} + - name: Setup Node uses: actions/setup-node@v4 with: node-version: '18' - - name: Install Node dependencies - run: | - git submodule update --init --recursive - npm ci + + - name: Restore npm cache + uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }} + + - name: Cache Playwright browsers + uses: actions/cache@v4 + id: playwright-cache + with: + path: ~/.cache/ms-playwright + key: ${{ runner.OS }}-playwright-${{ hashFiles('**/package-lock.json') }} + - name: Install Playwright browsers + if: steps.playwright-cache.outputs.cache-hit != 'true' run: npm run e2e:install + - name: Build o1js and prepare the web server run: | npm run build:web npm run e2e:prepare-server + - name: Execute E2E tests run: npm run test:e2e + - name: Upload E2E test artifacts uses: actions/upload-artifact@v4 continue-on-error: true @@ -80,19 +242,24 @@ jobs: if: github.ref == 'refs/heads/main' timeout-minutes: 180 runs-on: ubuntu-latest - needs: [Build-And-Test-Server, Build-And-Test-Web] + needs: [Build-And-Test-Server, Run-Unit-Tests, Build-And-Test-Web] steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Restore repository + uses: actions/cache@v4 + with: + path: . + key: repo-${{ github.sha }} + - name: Setup Node uses: actions/setup-node@v4 with: node-version: '18' + - name: Build o1js run: | - git submodule update --init --recursive npm ci - npm run build + npm run prepublishOnly + - name: Publish to NPM if version has changed uses: JS-DevTools/npm-publish@v3 with: @@ -105,21 +272,26 @@ jobs: if: github.ref == 'refs/heads/main' timeout-minutes: 180 runs-on: ubuntu-latest - needs: [Build-And-Test-Server, Build-And-Test-Web] + needs: [Build-And-Test-Server, Run-Unit-Tests, Build-And-Test-Web] steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Restore repository + uses: actions/cache@v4 + with: + path: . + key: repo-${{ github.sha }} + - name: Setup Node uses: actions/setup-node@v4 with: node-version: '18' + - name: Build mina-signer run: | - git submodule update --init --recursive npm ci cd src/mina-signer npm ci npm run prepublishOnly + - name: Publish to NPM if version has changed uses: JS-DevTools/npm-publish@v3 with: @@ -127,4 +299,4 @@ jobs: package: './src/mina-signer/package.json' strategy: upgrade env: - INPUT_TOKEN: ${{ secrets.NPM_TOKEN }} + INPUT_TOKEN: ${{ secrets.NPM_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 1d266fa3c9..6bd4433615 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -16,7 +16,7 @@ jobs: node-version: '18' - name: Run typedoc run: | - git submodule update --init --recursive + GIT_LFS_SKIP_SMUDGE=1 git submodule update --init --recursive npm ci npx typedoc --tsconfig tsconfig.node.json src/index.ts - name: Deploy diff --git a/.github/workflows/pkg-pr-new-publish.yml b/.github/workflows/pkg-pr-new-publish.yml index f075ad0da6..d4fc74c606 100644 --- a/.github/workflows/pkg-pr-new-publish.yml +++ b/.github/workflows/pkg-pr-new-publish.yml @@ -24,7 +24,7 @@ jobs: node-version: ${{ matrix.node }} - name: Build o1js and mina-signer run: | - git submodule update --init --recursive + GIT_LFS_SKIP_SMUDGE=1 git submodule update --init --recursive npm ci npm run build cd src/mina-signer diff --git a/run-ci-tests.sh b/run-ci-tests.sh index f005d78e0f..c5153b514e 100755 --- a/run-ci-tests.sh +++ b/run-ci-tests.sh @@ -34,15 +34,6 @@ case $TEST_TYPE in ./run src/examples/zkapps/dex/happy-path-with-proofs.ts --bundle ;; -"Unit tests") - echo "Running unit tests" - cd src/mina-signer - npm run build - cd ../.. - npm run test:unit - npm run test - ;; - "Verification Key Regression Check") echo "Running Regression checks" ./run ./tests/vk-regression/vk-regression.ts --bundle