Merge branch 'master' into gd/noir_issue_4330

* master: fix: Broadcasting unconstrained function with empty sibling (#5429) feat: Sync from noir (#5416) refactor: Meld flavor and and circuit builder modules (#5406) feat: less earthly runners + e2e GA runners, bb bench (#5356) fix: Track class registered count in tx stats (#5417)
AztecProtocol · Mar 25, 2024 · 7447259 · 7447259
2 parents ffe5712 + 933145e
commit 7447259
Show file tree

Hide file tree

Showing 438 changed files with 4,678 additions and 3,130 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -256,7 +256,7 @@ jobs:
       - *setup_env
       - run:
           name: "Test"
-          command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 proof_system_tests
+          command: cond_spot_run_test barretenberg-x86_64-linux-clang-assert 32 ./scripts/run_tests 1 stdlib_circuit_builders_tests
           aztec_manifest_key: barretenberg-x86_64-linux-clang-assert
 
   barretenberg-stdlib-plonk-recursion-ultra-tests:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,16 +6,15 @@ on:
   workflow_dispatch: {}
 
 jobs:
-  e2e:
+  # there's a lot of x86 tasks - let's split out the build step
+  build-x86:
     runs-on: ubuntu-latest
     env:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
-    # TODO currently names are coupled to platform
-    strategy: { matrix: { environment: [x86, arm], test: [e2e-escrow-contract, e2e-account-contracts] } }
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      group: ${{ matrix.test }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.environment }}
-      cancel-in-progress: true
+      group: build-${{ github.event.pull_request.number || github.ref_name }}-x86
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
     steps:
       - uses: earthly/actions-setup@v1
         with:
@@ -29,38 +28,171 @@ jobs:
 
       - name: Setup
         working-directory: ./scripts
-        run: ./setup_env.sh ${{ matrix.environment }} ${{ secrets.DOCKERHUB_PASSWORD }} ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
 
       - name: Test
-        working-directory: ./yarn-project/end-to-end
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr'
+        working-directory: ./yarn-project
+        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
         run: |
-          # TODO put in script
-          if [ "${{ matrix.environment }}" == "arm" ]; then
-            PLATFORM=linux/arm64
-          elif [ "${{ matrix.environment }}" == "x86" ]; then
-            PLATFORM=linux/amd64
-          fi
-          earthly sat --org aztec launch --size 4xlarge --platform $PLATFORM build-${{github.actor}}-${{ matrix.environment }} || true
-          if [ ${{ github.ref_name }} = master ] ; then
-            # update the remote cache
-            export EARTHLY_PUSH=true
-          fi
-          # TODO need to use more SAVE IMAGE --cache-hint and explicit BUILD statements for remote-cache to work well but then it should read artifacts from master done by all runners
-          earthly -P --no-output --org aztec --remote-cache=aztecprotocol/cache:${{matrix.test}} --sat build-${{github.actor}}-${{ matrix.environment }} +${{ matrix.test }}
+          # push to registry
+          earthly-cloud build x86 --push +build-end-to-end
 
+  # all the end-to-end integration tests for aztec
+  e2e-arm:
+    runs-on: ubuntu-latest
+    env:
+      EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
+    strategy:
+      matrix:
+        test:
+          - e2e-card-game
+          - e2e-crowdfunding-and-claim
+    # cancel if reran on same PR if exists, otherwise if on same commit
+    concurrency:
+      group: ${{ matrix.test }}-${{ github.event.pull_request.number || github.ref_name }}-arm
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
+    steps:
+      - uses: earthly/actions-setup@v1
+        with:
+          version: v0.8.5
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
+
+      - name: Setup
+        working-directory: ./scripts
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+
+      - name: Test
+        working-directory: ./yarn-project/end-to-end/
+        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
+        # We don't do much on arm, just run it on their builder
+        run: 
+          earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_remote=true
+
+  # all the end-to-end integration tests for aztec
+  e2e-x86:
+    needs: build-x86
+    runs-on: ubuntu-latest
+    env:
+      EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
+    strategy:
+      matrix:
+        test:
+          - e2e-state-vars
+          - e2e-block-building
+          - e2e-nested-contract
+          - e2e-static-calls
+          - e2e-delegate-calls
+          - e2e-non-contract-account
+          - e2e-cross-chain-messaging
+          - e2e-crowdfunding-and-claim
+          - e2e-public-cross-chain-messaging
+          - e2e-public-to-private-messaging
+          - e2e-account-contracts
+          - e2e-escrow-contract
+          - e2e-inclusion-proofs-contract
+          - e2e-pending-note-hashes-contract
+          - e2e-ordering
+          - uniswap-trade-on-l1-from-l2
+          - integration-archiver-l1-to-l2
+          - integration-l1-publisher
+          - e2e-cli
+          - e2e-persistence
+          # - e2e-browser # TODO failing
+          - e2e-card-game
+          - e2e-avm-simulator
+          - e2e-fees
+          - e2e-dapp-subscription
+          - pxe
+          - cli-docs-sandbox
+          # - e2e-docs-examples # TODO failing
+    # cancel if reran on same PR if exists, otherwise if on same commit
+    concurrency:
+      group: ${{ matrix.test }}-${{ github.event.pull_request.number || github.ref_name }}-x86
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
+    steps:
+      - uses: earthly/actions-setup@v1
+        with:
+          version: v0.8.5
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
+
+      - name: Setup
+        working-directory: ./scripts
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+
+      - name: Pull E2E Images
+        working-directory: ./barretenberg/cpp/
+        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
+        run: |
+          docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD)
+          docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD)
+          docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest
+          docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest
+
+      - name: Test
+        working-directory: ./yarn-project/end-to-end/
+        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
+        run: |
+          earthly-cloud build x86 --no-output +${{ matrix.test }}
+
+  # barretenberg (prover) native tests
   bb-native-tests:
     runs-on: ubuntu-latest
     env:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
-    # run for both x86_64 and arm64
-    # TODO currently names are coupled to platform
     strategy: { matrix: { environment: [x86, arm] } }
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      group: bb-native-tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.environment }}
-      cancel-in-progress: true
+      group: bb-native-tests-${{ github.event.pull_request.number || github.ref_name }}-${{ matrix.environment }}
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
+    steps:
+      - uses: earthly/actions-setup@v1
+        with:
+          version: v0.8.5
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
+
+      - name: Setup
+        working-directory: ./scripts
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+
+      - name: Test
+        working-directory: ./barretenberg/cpp/
+        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
+        run: |
+          earthly-cloud build ${{ matrix.environment }} --no-output +test
+
+  # All benchmarks, purposefully ran sequential on a machine
+  # they should use parallelism within the benchmark, but only one thing should run at a time
+  # for accurate results
+  # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them.
+  bench:
+    runs-on: ubuntu-latest
+    env:
+      EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
+    # cancel if reran on same PR if exists, otherwise if on same commit
+    concurrency:
+      # TODO no longer exclusive.. revisit this
+      group: ${{ github.actor }}-x86
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
     steps:
+      - uses: earthly/actions-setup@v1
+        with:
+          version: v0.8.5
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -69,22 +201,14 @@ jobs:
 
       - name: Setup
         working-directory: ./scripts
-        run: ./setup_env.sh ${{ matrix.environment }} ${{ secrets.DOCKERHUB_PASSWORD }} ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
-
-      - name: Build and test
-        working-directory: ./barretenberg/cpp
-        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr'
-        run:  |
-          # TODO put in script
-          if [ "${{ matrix.environment }}" == "arm" ]; then
-            PLATFORM=linux/arm64
-          elif [ "${{ matrix.environment }}" == "x86" ]; then
-            PLATFORM=linux/amd64
-          fi
-          earthly sat --org aztec launch --size 4xlarge --platform $PLATFORM build-${{github.actor}}-${{ matrix.environment }} || true
-          if [ ${{ github.ref_name }} = master ] ; then
-            # update the remote cache
-            export EARTHLY_PUSH=true
-          fi
-          # TODO need to use more SAVE IMAGE --cache-hint and explicit BUILD statements for remote-cache to work well but then it should read artifacts from master done by all runners
-          earthly -P --no-output --org aztec --remote-cache=aztecprotocol/cache:bb-native-tests --sat build-${{github.actor}}-${{ matrix.environment }} +test
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }}
+
+      - name: Client IVC Bench
+        working-directory: ./barretenberg/cpp/
+        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
+        run: earthly-cloud bench x86 --no-output +bench-client-ivc
+
+      - name: Ultrahonk Bench
+        working-directory: ./barretenberg/cpp/
+        if: github.actor == 'ludamad' || github.actor == 'charlielye' || github.actor == 'sklppy88' || github.actor == 'alexghr' || github.actor == 'benesjan' || github.actor == 'spalladino' || github.actor == 'just-mitch'
+        run: earthly-cloud bench x86 --no-output +bench-ultra-honk
diff --git a/Earthfile b/Earthfile
@@ -3,19 +3,18 @@ FROM ubuntu:lunar
 
 build-ci:
     BUILD ./avm-transpiler/+build
-    BUILD ./barretenberg/cpp/+build-release
+    BUILD ./barretenberg/cpp/+preset-release
     BUILD ./barretenberg/cpp/+preset-wasm
-    BUILD ./barretenberg/cpp/+build-gcc
-    BUILD ./barretenberg/cpp/+build-fuzzing
-    BUILD ./barretenberg/cpp/+build-clang-assert
-    BUILD ./barretenberg/cpp/+test-clang-format
+    BUILD ./barretenberg/cpp/+preset-gcc
+    BUILD ./barretenberg/cpp/+preset-fuzzing
+    BUILD ./barretenberg/cpp/+preset-clang-assert
     BUILD ./barretenberg/cpp/+test-clang-format
     BUILD ./boxes/+build
     BUILD ./noir/+packages
     BUILD ./noir/+nargo
     BUILD ./noir-projects/+build
-    BUILD ./yarn-project/+build
-    BUILD +test-end-to-end
+    BUILD ./yarn-project/+end-to-end-minimal
+    BUILD ./yarn-project/+aztec
 
 build-ci-small:
     BUILD ./yarn-project/end-to-end/+e2e-escrow-contract

diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile
@@ -12,7 +12,12 @@ RUN apt-get update && apt-get install -y git
 
 COPY --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml .
 
+source:
+  # for debugging rebuilds
+  RUN echo CONTENT HASH $(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}') | tee .content-hash
+
 build:
+    FROM +source
     RUN ./scripts/bootstrap_native.sh
     SAVE ARTIFACT target/release/avm-transpiler avm-transpiler
 

diff --git a/barretenberg/cpp/.earthlyignore b/barretenberg/cpp/.earthlyignore
@@ -0,0 +1 @@
+.gitignore
diff --git a/barretenberg/cpp/.gitignore b/barretenberg/cpp/.gitignore
@@ -1,7 +1,7 @@
 .cache/
 build*/
 src/wasi-sdk*
-src/barretenberg/proof_system/proving_key/fixtures
+src/barretenberg/plonk_honk_shared/proving_key/fixtures
 src/barretenberg/rollup/proofs/*/fixtures
 srs_db/*/*/transcript*
 srs_db/*/bn254_g*
@@ -12,3 +12,4 @@ acir_tests
 go*.tar.gz
 barretenberg_modules.dot
 barretenberg_modules.png
+src/barretenberg/bb/config.hpp
diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json
@@ -74,7 +74,7 @@
       "name": "clang16-assert",
       "displayName": "Build with Clang-16 using RelWithAssert",
       "description": "Build with globally installed Clang-16 in release with ASSERTs mode",
-      "inherits": "default",
+      "inherits": "clang16",
       "environment": {
         "CMAKE_BUILD_TYPE": "RelWithAssert"
       }