feat: initial Earthly CI

.circleci/config.yml

@@ -1313,7 +1313,10 @@ workflows:
       - barretenberg-stdlib-tests: *bb_test
       - barretenberg-stdlib-recursion-ultra-tests: *bb_test
       - barretenberg-acir-tests-bb: *bb_acir_tests
-      - barretenberg-acir-tests-bb-sol: *bb_acir_tests
+      - barretenberg-acir-tests-bb-sol:
+          requires:
+            - barretenberg-x86_64-linux-clang-sol
+          <<: *bb_acir_tests
       - barretenberg-docs: *defaults
       - bb-js:
           requires:

.github/workflows/ci.yml

@@ -0,0 +1,70 @@
+# Runs the entire new CI mode.
+name: Run CI with Earthly
+on:
+  push:
+    branches:
+      - master
+  pull_request: {}
+  workflow_dispatch: {}
+jobs:
+  ci-x86:
+    runs-on: ubuntu-latest
+    # TODO enable after stress tests
+    # concurrency:
+    #   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-x86
+    #   cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
+      - name: setup
+        run: |
+          mkdir -p ~/.ssh
+          echo DOCKER_HOST=ssh://build-instance-x86.aztecprotocol.com >> $GITHUB_ENV
+          echo ${{ secrets.DOCKERHUB_PASSWORD}} | docker login -u aztecprotocolci --password-stdin
+          echo ${{ secrets.BUILD_INSTANCE_SSH_KEY }} | base64 -d > ~/.ssh/build_instance_key
+          chmod 600 ~/.ssh/build_instance_key
+          cat > ~/.ssh/config <<EOF
+          IdentityFile ~/.ssh/build_instance_key
+          StrictHostKeyChecking no
+          User ubuntu
+          EOF
+      # - name: ensure-base-images
+      #   run: |
+      #     scripts/earthly --push ./foundry/+build
+      #      scripts/earthly --push ./barretenberg/cpp/+build-wasi-sdkV
+      - name: ci
+        run: scripts/earthly --exec-stats +build-ci-small
+
+  ci-arm:
+    runs-on: ubuntu-latest
+    # TODO enable after stress tests
+    # concurrency:
+    #   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-arm
+    #   cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
+      - name: setup
+        run: |
+          mkdir -p ~/.ssh
+          echo DOCKER_HOST=ssh://build-instance-arm.aztecprotocol.com >> $GITHUB_ENV
+          echo ${{ secrets.DOCKERHUB_PASSWORD}} | docker login -u aztecprotocolci --password-stdin
+          echo ${{ secrets.BUILD_INSTANCE_SSH_KEY }} | base64 -d > ~/.ssh/build_instance_key
+          chmod 600 ~/.ssh/build_instance_key
+          cat > ~/.ssh/config <<EOF
+          IdentityFile ~/.ssh/build_instance_key
+          StrictHostKeyChecking no
+          User ubuntu
+          EOF
+      - name: ensure-base-images
+        run: |
+          scripts/earthly --push ./foundry/+build
+          # scripts/earthly --push ./barretenberg/cpp/+build-wasi-sdk
+      - name: ci
+        run: scripts/earthly --exec-stats +build-ci-small

.gitignore

@@ -1,6 +1,7 @@
 dest
 node_modules
 .cache
+scripts/.earthly
 .pnp.cjs
 .pnp.loader.mjs
 build/
@@ -16,4 +17,4 @@ cmake-build-debug
 .graphite*
 .DS_Store
 
-**/*.dockerignore
+**/*.dockerignore

Earthfile

@@ -0,0 +1,31 @@
+VERSION 0.8
+FROM ubuntu:lunar
+
+build-ci:
+    BUILD ./avm-transpiler/+build
+    BUILD ./barretenberg/cpp/+build-release
+    BUILD ./barretenberg/cpp/+preset-wasm
+    BUILD ./barretenberg/cpp/+build-gcc
+    BUILD ./barretenberg/cpp/+build-fuzzing
+    BUILD ./barretenberg/cpp/+build-clang-assert
+    BUILD ./barretenberg/cpp/+test-clang-format
+    BUILD ./barretenberg/cpp/+test-clang-format
+    BUILD ./boxes/+build
+    BUILD ./noir/+packages
+    BUILD ./noir/+nargo
+    BUILD ./noir-projects/+build
+    BUILD ./yarn-project/+build
+    BUILD +test-end-to-end
+
+build-ci-small:
+    BUILD ./yarn-project/end-to-end/+e2e-escrow-contract
+
+build:
+    # yarn-project has the entry point to Aztec
+    BUILD ./yarn-project/+build
+
+test-end-to-end:
+    BUILD ./yarn-project/end-to-end/+test-all
+
+bench:
+  RUN echo hi

README.md

@@ -45,8 +45,9 @@ It is faster to debug CI failures within a persistent ssh session compared to pu
 
 ```bash
 cd project
-./build-system/scripts/setup_env "$(git rev-parse HEAD)" "" "" ""
+./build-system/scripts/setup_env "$(git rev-parse HEAD)" "" https://github.com/AztecProtocol/aztec-packages
 source /tmp/.bash_env*
+set +euo
 {start testing your CI commands here}
 ```
 
@@ -76,3 +77,18 @@ Recovering if the sync is not happening with basic pull commands:
   this needs to exist in the branch we push to, and have the same content as our base. This is similar to submodules, except instead of pointing to the final state of the module, it points to the last commit we have sync'd from, for purposes of commit replay. This can be fixed to match the commit in master after merges.
 - manually editing the parent variable in noir/noir-repo/.gitrepo: this is the parent of the last sync commit on aztec side. If you get errors with a commit not being found in the upstream repo, and the commit mentioned is not the commit variable above, it might indicate this is somehow incorrect. This can happen when commit content is ported without its history, e.g. squashes
 - use pull --force ONLY where you would use git reset. That is, if you really want to match some upstream noir for a purpose its fine, but you'll lose local changes (if any)
+
+
+## Earthly
+
+Earthly is a reproducible build tool that aims to combine the functionality of Docker, Makefiles and BASH.
+Non-build earthly targets should start with 'test', 'run', or 'bench' as a general rule (but not hard rule) while builds can be nouns or start with build-.
+If something is a bundle of targets for CI, we can do e.g. build-ci, test-ci etc.
+See barretenberg/cpp/Earthfile for an example of a fairly involved Earthfile that can be used for inspiration.
+Earthly docs https://docs.earthly.dev/ are extensive and show the various build patterns. 
+
+In a nutshell:
+- Docker-like syntax defines all builds. We lean on docker heavily for when to rebuild and cache, and how to run in a reproducible manner.
+- It supports modularization of the build manifest into multiple directories that can be imported. Simple functions and conditional logic can be used.
+- We provide two modes, one for CI by passing --ci and one for local with incremental builds.
+- We do NOT provide a native execution story for anything but Linux and WASM currently. 

avm-transpiler/Earthfile

@@ -0,0 +1,23 @@
+VERSION 0.8
+IMPORT ../noir AS noir
+# we rely on noir source, which this image has
+FROM noir+nargo
+
+# move noir contents to /build/noir
+RUN mv /build /noir && mkdir /build && mv /noir /build
+# work in avm-transpiler
+WORKDIR /build/avm-transpiler
+
+RUN apt-get update && apt-get install -y git
+
+COPY --keep-ts --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml .
+
+build:
+    RUN ./scripts/bootstrap_native.sh
+    SAVE ARTIFACT target/release/avm-transpiler avm-transpiler
+
+run: 
+    #TODO needed?
+    FROM ubuntu:focal
+    COPY +build/avm-transpiler /build/avm-transpiler
+    ENTRYPOINT ["sh", "-c"]

barretenberg/README.md

@@ -257,8 +257,6 @@ CI will automatically run integration tests against Aztec. It is located in the
 
 ### Integration tests with Aztec in Barretenberg Standalone Repo
 
-CI will automatically run integration tests against Aztec's circuits which live [here](https://github.com/AztecProtocol/aztec-packages/tree/master/circuits). To change which Aztec branch or commit for CI to test against, modify [`.aztec-packages-commit`](./cpp/.aztec-packages-commit).
-
 When working on a PR, you may want to point this file to a different Aztec branch or commit, but then it should probably be pointed back to master before merging.
 
 ### Testing locally in docker

barretenberg/cpp/CMakePresets.json

@@ -114,7 +114,8 @@
         "CC": "gcc-13",
         "CXX": "g++-13"
       }
-    },    {
+    },
+    {
       "name": "bench",
       "displayName": "Build benchmarks",
       "description": "Build default preset but with a special benchmark directory",
@@ -196,11 +197,11 @@
       }
     },
     {
-      "name": "op-count-track",
+      "name": "op-count",
       "displayName": "Release build with operation counts",
       "description": "Build with op counting",
       "inherits": "clang16",
-      "binaryDir": "build-op-count-track",
+      "binaryDir": "build-op-count",
       "environment": {
         "CXXFLAGS": "-DBB_USE_OP_COUNT -DBB_USE_OP_COUNT_TRACK_ONLY"
       }
@@ -246,7 +247,7 @@
       "generator": "Ninja",
       "toolchainFile": "cmake/toolchains/wasm32-wasi.cmake",
       "environment": {
-        "WASI_SDK_PREFIX": "${sourceDir}/src/wasi-sdk-20.0",
+        "WASI_SDK_PREFIX": "${sourceDir}/src/wasi-sdk",
         "CC": "$env{WASI_SDK_PREFIX}/bin/clang",
         "CXX": "$env{WASI_SDK_PREFIX}/bin/clang++",
         "AR": "$env{WASI_SDK_PREFIX}/bin/llvm-ar",
@@ -279,7 +280,6 @@
       "inherits": "wasm",
       "binaryDir": "build-wasm-threads",
       "environment": {
-        "WASI_SDK_PREFIX": "${sourceDir}/src/wasi-sdk-20.0+threads",
         "CMAKE_BUILD_TYPE": "Release"
       },
       "cacheVariables": {
@@ -345,9 +345,9 @@
       "configurePreset": "op-count-time"
     },
     {
-      "name": "op-count-track",
+      "name": "op-count",
       "inherits": "default",
-      "configurePreset": "op-count-track"
+      "configurePreset": "op-count"
     },
     {
       "name": "clang16-dbg",
@@ -389,7 +389,7 @@
       "inherits": "clang16",
       "configurePreset": "fuzzing"
     },
-{
+    {
       "name": "fuzzing-asan",
       "inherits": "clang16-dbg",
       "configurePreset": "fuzzing-asan"
@@ -518,7 +518,7 @@
       "inherits": "default",
       "configurePreset": "fuzzing"
     },
-{
+    {
       "name": "fuzzing-asan",
       "inherits": "clang16-dbg",
       "configurePreset": "fuzzing-asan"