feat: less earthly runners + e2e GA runners, bb bench

.github/workflows/ci.yml

@@ -6,16 +6,20 @@ on:
   workflow_dispatch: {}
 
 jobs:
-  e2e:
+  # all the end-to-end integration tests for aztec
+  e2e-arm:
     runs-on: ubuntu-latest
     env:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
-    # TODO currently names are coupled to platform
-    strategy: { matrix: { environment: [x86, arm], test: [e2e-escrow-contract, e2e-account-contracts] } }
+    strategy:
+      matrix:
+        test:
+          - e2e-card-game
+          - e2e-crowdfunding-and-claim
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      group: ${{ matrix.test }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.environment }}
-      cancel-in-progress: true
+      group: ${{ matrix.test }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-arm
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
     steps:
       - uses: earthly/actions-setup@v1
         with:
@@ -29,37 +33,86 @@ jobs:
 
       - name: Setup
         working-directory: ./scripts
-        run: ./setup_env.sh ${{ matrix.environment }} ${{ secrets.DOCKERHUB_PASSWORD }} ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Test
-        working-directory: ./yarn-project/end-to-end
-        run: |
-          # TODO put in script
-          if [ "${{ matrix.environment }}" == "arm" ]; then
-            PLATFORM=linux/arm64
-          elif [ "${{ matrix.environment }}" == "x86" ]; then
-            PLATFORM=linux/amd64
-          fi
-          earthly sat --org aztec launch --size 4xlarge --platform $PLATFORM build-${{github.actor}}-${{ matrix.environment }} || true
-          if [ ${{ github.ref_name }} = master ] ; then
-            # update the remote cache
-            export EARTHLY_PUSH=true
-          fi
-          # TODO need to use more SAVE IMAGE --cache-hint and explicit BUILD statements for remote-cache to work well but then it should read artifacts from master done by all runners
-          earthly -P --no-output --org aztec --remote-cache=aztecprotocol/cache:${{matrix.test}} --sat build-${{github.actor}}-${{ matrix.environment }} +${{ matrix.test }}
+        working-directory: ./yarn-project/end-to-end/
+        run: earthly-cloud ${{ github.actor }} arm +${{ matrix.test }}
 
+  # all the end-to-end integration tests for aztec
+  e2e-x86:
+    runs-on: ubuntu-latest
+    env:
+      EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
+    strategy:
+      matrix:
+        test:
+          - e2e-state-vars
+          - e2e-block-building
+          - e2e-nested-contract
+          - e2e-static-calls
+          - e2e-delegate-calls
+          - e2e-non-contract-account
+          - e2e-cross-chain-messaging
+          - e2e-crowdfunding-and-claim
+          - e2e-public-cross-chain-messaging
+          - e2e-public-to-private-messaging
+          - e2e-account-contracts
+          - e2e-escrow-contract
+          - e2e-inclusion-proofs-contract
+          - e2e-pending-note-hashes-contract
+          - e2e-ordering
+          - uniswap-trade-on-l1-from-l2
+          - integration-archiver-l1-to-l2
+          - integration-l1-publisher
+          - e2e-cli
+          - e2e-persistence
+          - e2e-browser
+          - e2e-card-game
+          - e2e-avm-simulator
+          - e2e-fees
+          - e2e-dapp-subscription
+          - pxe
+          - cli-docs-sandbox
+          - e2e-docs-examples
+    # cancel if reran on same PR if exists, otherwise if on same commit
+    concurrency:
+      group: ${{ matrix.test }}-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-x86
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
+    steps:
+      - uses: earthly/actions-setup@v1
+        with:
+          version: v0.8.5
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
+
+      - name: Setup
+        working-directory: ./scripts
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Test
+        working-directory: ./yarn-project/end-to-end/
+        run: earthly-cloud ${{ github.actor }} x86 +${{ matrix.test }}
+
+  # barretenberg (prover) native tests
   bb-native-tests:
     runs-on: ubuntu-latest
     env:
       EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
-    # run for both x86_64 and arm64
-    # TODO currently names are coupled to platform
     strategy: { matrix: { environment: [x86, arm] } }
     # cancel if reran on same PR if exists, otherwise if on same commit
     concurrency:
-      group: bb-native-tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.environment }}
-      cancel-in-progress: true
+      group: bb-native-tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ matrix.environment }}
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
     steps:
+      - uses: earthly/actions-setup@v1
+        with:
+          version: v0.8.5
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -68,21 +121,45 @@ jobs:
 
       - name: Setup
         working-directory: ./scripts
-        run: ./setup_env.sh ${{ matrix.environment }} ${{ secrets.DOCKERHUB_PASSWORD }} ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
-
-      - name: Build and test
-        working-directory: ./barretenberg/cpp
-        run:  |
-          # TODO put in script
-          if [ "${{ matrix.environment }}" == "arm" ]; then
-            PLATFORM=linux/arm64
-          elif [ "${{ matrix.environment }}" == "x86" ]; then
-            PLATFORM=linux/amd64
-          fi
-          earthly sat --org aztec launch --size 4xlarge --platform $PLATFORM build-${{github.actor}}-${{ matrix.environment }} || true
-          if [ ${{ github.ref_name }} = master ] ; then
-            # update the remote cache
-            export EARTHLY_PUSH=true
-          fi
-          # TODO need to use more SAVE IMAGE --cache-hint and explicit BUILD statements for remote-cache to work well but then it should read artifacts from master done by all runners
-          earthly -P --no-output --org aztec --remote-cache=aztecprotocol/cache:bb-native-tests --sat build-${{github.actor}}-${{ matrix.environment }} +test
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Test
+        working-directory: ./barretenberg/cpp/
+        run: earthly-cloud ${{ github.actor }} ${{ matrix.environment }} +test
+
+  # All benchmarks, purposefully ran sequential on a machine
+  # they should use parallelism within the benchmark, but only one thing should run at a time
+  # for accurate results
+  # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them.
+  bench:
+    runs-on: ubuntu-latest
+    env:
+      EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }}
+    # cancel if reran on same PR if exists, otherwise if on same commit
+    concurrency:
+      # need this to match the user's bench runner
+      # earthly satellites can't enforce parallelism==1 so we do it through github
+      group: bench-${{ github.actor }}
+      cancel-in-progress: ${{ github.ref_name != 'master' }}
+    steps:
+      - uses: earthly/actions-setup@v1
+        with:
+          version: v0.8.5
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
+
+      - name: Setup
+        working-directory: ./scripts
+        run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Client IVC Bench
+        working-directory: ./barretenberg/cpp/
+        run: earthly-cloud-bench ${{ github.actor }} x86 +bench-client-ivc
+
+      - name: Ultrahonk Bench
+        working-directory: ./barretenberg/cpp/
+        run: earthly-cloud-bench ${{ github.actor }} x86 +bench-ultra-honk

Earthfile

@@ -3,19 +3,18 @@ FROM ubuntu:lunar
 
 build-ci:
     BUILD ./avm-transpiler/+build
-    BUILD ./barretenberg/cpp/+build-release
+    BUILD ./barretenberg/cpp/+preset-release
     BUILD ./barretenberg/cpp/+preset-wasm
-    BUILD ./barretenberg/cpp/+build-gcc
-    BUILD ./barretenberg/cpp/+build-fuzzing
-    BUILD ./barretenberg/cpp/+build-clang-assert
-    BUILD ./barretenberg/cpp/+test-clang-format
+    BUILD ./barretenberg/cpp/+preset-gcc
+    BUILD ./barretenberg/cpp/+preset-fuzzing
+    BUILD ./barretenberg/cpp/+preset-clang-assert
     BUILD ./barretenberg/cpp/+test-clang-format
     BUILD ./boxes/+build
     BUILD ./noir/+packages
     BUILD ./noir/+nargo
     BUILD ./noir-projects/+build
-    BUILD ./yarn-project/+build
-    BUILD +test-end-to-end
+    BUILD ./yarn-project/+end-to-end-minimal
+    BUILD ./yarn-project/+aztec
 
 build-ci-small:
     BUILD ./yarn-project/end-to-end/+e2e-escrow-contract

barretenberg/cpp/Earthfile

@@ -53,6 +53,10 @@ get-wasi-sdk:
     FROM aztecprotocol/cache:wasi-sdk-threads-v21.0-$TARGETARCH
     SAVE ARTIFACT wasi-sdk
 
+wasmtime:
+    RUN curl https://wasmtime.dev/install.sh -sSf | bash
+    SAVE ARTIFACT /root/.wasmtime/bin/wasmtime
+
 source:
     # cpp source
     COPY --keep-ts --dir src/barretenberg src/CMakeLists.txt src
@@ -61,7 +65,7 @@ source:
 
 preset-release-assert-all:
     FROM +source
-    RUN cmake --preset clang16 -DCMAKE_BUILD_TYPE=RelWithAssert && cmake --build --preset clang16
+    DO +RUN_CMAKE --configure="--preset clang16 -DCMAKE_BUILD_TYPE=RelWithAssert" --build=""
     SAVE ARTIFACT bin
 
 preset-release:
@@ -132,7 +136,7 @@ test:
     BUILD +test-clang-format
     FROM +preset-release-assert-all
     COPY --dir ./srs_db/+build/. srs_db
-    RUN cd build && GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure
+    RUN GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure
 
 # Functions
 RUN_CMAKE:
@@ -159,33 +163,35 @@ BENCH_RELEASE:
     ARG target
     ARG args
     FROM +preset-release
-    DO +BUILD --configure="--preset clang16" --build="--target $target"
-    COPY --dir ./srs_db/+build/. srs_db
-    RUN cd build && ./bin/$target $args
+    DO +RUN_CMAKE --configure="--preset clang16" --build="--target $target"
+    COPY --dir ./srs_db/+build/. ../srs_db
+    RUN ./bin/$target $args
 
 BENCH_OP_COUNT_TIME:
     FUNCTION
     ARG target
     ARG args
     FROM +preset-op-count-time
     DO +RUN_CMAKE --configure="--preset op-count-time" --build="--target $target"
-    COPY --dir ./srs_db:+build/. srs_db
-    RUN cd build && ./bin/$target $args
+    COPY --dir ./srs_db:+build/. ../srs_db
+    RUN ./bin/$target $args
 
 BENCH_OP_COUNT:
     FUNCTION
     ARG target
     ARG args
     FROM +preset-op-count
     DO +RUN_CMAKE --configure="--preset op-count" --build="--target $target"
-    COPY --dir ./srs_db/+build/. srs_db
-    RUN cd build && ./bin/$target $args
+    COPY --dir ./srs_db/+build/. ../srs_db
+    RUN ./bin/$target $args
 
 BENCH_WASM:
     FUNCTION
     ARG target
     ARG args
     FROM +preset-wasm
     DO +RUN_CMAKE --configure="--preset wasm-threads" --build="--target $target"
-    COPY --dir ./srs_db/+build/. srs_db
-    RUN cd build && ~/.wasmtime/bin/wasmtime run --env HARDWARE_CONCURRENCY=8 -Wthreads=y -Sthreads=y --dir=.. ./bin/$target $args
+    COPY --dir ./srs_db/+build/. ../srs_db
+    # install
+    COPY +wasmtime/wasmtime /usr/bin/wasmtime
+    RUN wasmtime run --env HARDWARE_CONCURRENCY=8 -Wthreads=y -Sthreads=y --dir=.. ./bin/$target $args

scripts/earthly-cloud

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -eux
+
+USERNAME=$1
+ARCH=$2
+
+shift 2
+
+if [ "$ARCH" == "arm" ]; then
+  PLATFORM=linux/arm64
+elif [ "$ARCH" == "x86" ]; then
+  PLATFORM=linux/amd64
+fi
+earthly sat --org aztec launch --size 2xlarge --platform $PLATFORM build-$USERNAME-$ARCH || true
+earthly -P --no-output --org aztec --remote-cache=aztecprotocol/cache:bb-native-tests --sat build-$USERNAME-$ARCH $@

scripts/earthly-cloud-bench

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -eux
+
+USERNAME=$1
+ARCH=$2
+
+shift 2
+
+if [ "$ARCH" == "arm" ]; then
+  PLATFORM=linux/arm64
+elif [ "$ARCH" == "x86" ]; then
+  PLATFORM=linux/amd64
+fi
+earthly sat --org aztec launch --size 2xlarge --platform $PLATFORM bench-$USERNAME-$ARCH || true
+earthly -P --no-output --org aztec --remote-cache=aztecprotocol/cache:bb-native-tests --sat bench-$USERNAME-$ARCH $@

scripts/earthly-timed

@@ -1,4 +1,3 @@
-
 #!/usr/bin/env bash
 [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 
@@ -49,7 +48,7 @@ for arg in "$@"; do
     fi
 done
 if [ $INTERACTIVE = true ] ; then
-  $(dirname $0)/earthly $@
+  earthly $@
 else
-  $(dirname $0)/earthly $@ 2>&1 | add_timestamps >&2
+  earthly $@ 2>&1 | add_timestamps >&2
 fi