feat: stable deployments for spartan

scripts/ci/get_e2e_jobs.sh

@@ -7,7 +7,7 @@ cd "$(dirname "$0")"/../..
 BRANCH=$1
 LABELS=$2
 
-# Define the allow_list
+# Define the jobs that will run on every PR
 allow_list=(
   "e2e-2-pxes"
   "e2e-authwit"
@@ -26,6 +26,7 @@ allow_list=(
   "e2e-cheat-codes"
   "e2e-prover-fake-proofs"
   "e2e-lending-contract"
+  "kind-network-smoke"
 )
 
 # Add labels from input to the allow_list

spartan/aztec-network/templates/_helpers.tpl

@@ -67,13 +67,20 @@ http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }}
 {{- end -}}
 
 {{- define "aztec-network.otelCollectorMetricsEndpoint" -}}
-http://metrics-opentelemetry-collector.metrics:4318/v1/metrics
+{{- if .Values.telemetry.enabled -}}
+{{- if .Values.telemetry.otelCollectorEndpoint -}}
+{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/metrics
+{{- end -}}
+{{- end -}}
 {{- end -}}
 
 {{- define "aztec-network.otelCollectorTracesEndpoint" -}}
-http://metrics-opentelemetry-collector.metrics:4318/v1/traces
+{{- if .Values.telemetry.enabled -}}
+{{- if .Values.telemetry.otelCollectorEndpoint -}}
+{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/traces
+{{- end -}}
+{{- end -}}
 {{- end -}}
-
 
 
 {{- define "helpers.flag" -}}

spartan/aztec-network/templates/boot-node.yaml

@@ -31,11 +31,13 @@ spec:
                 sleep 5
               done
               echo "Ethereum node is ready!"
+              {{- if .Values.telemetry.enabled }}
               until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
                 echo "Waiting for OpenTelemetry collector..."
                 sleep 5
               done
               echo "OpenTelemetry collector is ready!"
+              {{- end }}
         - name: deploy-contracts
           image: {{ .Values.images.aztec.image }}
           command:
@@ -56,10 +58,11 @@ spec:
         - name: boot-node
           image: {{ .Values.images.aztec.image }}
           command:
+          # sleep to allow dns name to be resolvable
             [
               "/bin/bash",
               "-c",
-              "source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe",
+              "sleep 10 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe",
             ]
           livenessProbe:
             exec:

spartan/aztec-network/templates/l2-contracts.yaml

@@ -46,7 +46,7 @@ metadata:
 data:
   deploy-contracts.sh: |
     #!/bin/sh
-    set -e
+    set -ex
 
     # Run the deploy-l1-contracts command and capture the output
     output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --validators {{ join "," .Values.validator.validatorAddresses | quote }})

spartan/aztec-network/templates/prover-node.yaml

@@ -18,6 +18,30 @@ spec:
         app: prover-node
     spec:
       initContainers:
+        - name: wait-for-boot-node
+          image: {{ .Values.images.curl.image }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              until curl -s -X POST -H 'Content-Type: application/json' \
+                -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \
+                {{ include "aztec-network.ethereumHost" . }} | grep -q anvil; do
+                echo "Waiting for Ethereum node..."
+                sleep 5
+              done
+              echo "Ethereum node is ready!"
+              {{- if .Values.telemetry.enabled }}
+              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
+                echo "Waiting for OpenTelemetry collector..."
+                sleep 5
+              done
+              echo "OpenTelemetry collector is ready!"
+              {{- end }}
+              until curl --head --silent {{ include "aztec-network.bootNodeUrl" . }}/status; do
+                echo "Waiting for boot node..."
+                sleep 5
+              done
         - name: configure-prover-env
           image: "{{ .Values.images.aztec.image }}"
           imagePullPolicy: {{ .Values.images.aztec.pullPolicy }}
@@ -33,6 +57,7 @@ spec:
           env:
             - name: ETHEREUM_HOST
               value: {{ include "aztec-network.ethereumHost" . | quote }}
+
       containers:
         - name: prover-node
           image: "{{ .Values.images.aztec.image }}"

spartan/aztec-network/templates/pxe.yaml

@@ -17,6 +17,17 @@ spec:
         {{- include "aztec-network.selectorLabels" . | nindent 8 }}
         app: pxe
     spec:
+      initContainers:
+        - name: wait-for-boot-node
+          image: {{ .Values.images.curl.image }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              until curl --head --silent {{ include "aztec-network.bootNodeUrl" . }}/status; do
+                echo "Waiting for boot node..."
+                sleep 5
+              done
       containers:
         - name: pxe
           image: "{{ .Values.images.aztec.image }}"

spartan/aztec-network/templates/validator.yaml

@@ -21,6 +21,31 @@ spec:
       # We expect the validators to have already been added to the smart contract by this point - but this container still needs
       # to be run in order to get the values
       initContainers:
+        - name: wait-for-boot-node
+          image: {{ .Values.images.curl.image }}
+          command:
+            - /bin/sh
+            - -c
+            - |
+              until curl -s -X POST -H 'Content-Type: application/json' \
+                -d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \
+                {{ include "aztec-network.ethereumHost" . }} | grep -q anvil; do
+                echo "Waiting for Ethereum node..."
+                sleep 5
+              done
+              echo "Ethereum node is ready!"
+              {{- if .Values.telemetry.enabled }}
+              until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
+                echo "Waiting for OpenTelemetry collector..."
+                sleep 5
+              done
+              echo "OpenTelemetry collector is ready!"
+              {{- end }}
+              until curl --head --silent {{ include "aztec-network.bootNodeUrl" . }}/status; do
+                echo "Waiting for boot node..."
+                sleep 5
+              done
+
         - name: configure-validator-env
           image: "{{ .Values.images.aztec.image }}"
           imagePullPolicy: {{ .Values.images.aztec.pullPolicy }}
@@ -50,7 +75,7 @@ spec:
           command:
             - "/bin/bash"
             - "-c"
-            - "source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer"
+            - "sleep 10 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer"
           volumeMounts:
             - name: shared-volume
               mountPath: /shared

spartan/aztec-network/values.yaml

@@ -2,6 +2,10 @@ network:
   public: false
   enableBots: true
 
+telemetry:
+  enabled: false
+  otelCollectorEndpoint:
+
 images:
   aztec:
     image: aztecprotocol/aztec
@@ -31,13 +35,14 @@ bootNode:
     requests:
       memory: "2Gi"
       cpu: "200m"
-    limits:
-      memory: "4Gi"
-      cpu: "4"
   storage: "8Gi"
 
 validator:
   replicas: 1
+  validatorKeys:
+    - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
+  validatorAddresses:
+    - 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266
   service:
     p2pPort: 40400
     nodePort: 8080
@@ -54,9 +59,6 @@ validator:
     requests:
       memory: "2Gi"
       cpu: "200m"
-    limits:
-      memory: "28Gi"
-      cpu: "7"
   storage: "8Gi"
 
 proverNode:
@@ -71,9 +73,6 @@ proverNode:
     requests:
       memory: "2Gi"
       cpu: "200m"
-    limits:
-      memory: "120Gi"
-      cpu: "15"
   storage: "8Gi"
 
 pxe:
@@ -93,9 +92,6 @@ pxe:
     requests:
       memory: "2Gi"
       cpu: "200m"
-    limits:
-      memory: "4Gi"
-      cpu: "1"
 
 bot:
   logLevel: "debug"
@@ -124,9 +120,6 @@ bot:
     requests:
       memory: "2Gi"
       cpu: "200m"
-    limits:
-      memory: "4Gi"
-      cpu: "1"
 
 ethereum:
   replicas: 1
@@ -152,7 +145,4 @@ ethereum:
     requests:
       memory: "2Gi"
       cpu: "200m"
-    limits:
-      memory: "4Gi"
-      cpu: "1"
   storage: "8Gi"

spartan/aztec-network/values/16-validators.yaml

@@ -1,3 +1,13 @@
+##########
+# BEWARE #
+##########
+# You need to deploy the metrics helm chart before using this values file.
+# head to spartan/metrics and run `./install.sh`
+# (then `./forward.sh` if you want to see it)
+telemetry:
+  enabled: true
+  otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318
+
 bootNode:
   sequencer:
     minTxsPerBlock: 4

spartan/aztec-network/values/48-validators.yaml

@@ -1,3 +1,13 @@
+##########
+# BEWARE #
+##########
+# You need to deploy the metrics helm chart before using this values file.
+# head to spartan/metrics and run `./install.sh`
+# (then `./forward.sh` if you want to see it)
+telemetry:
+  enabled: true
+  otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318
+
 validator:
   debug: "aztec:*,-aztec:avm_simulator:*,-aztec:libp2p_service"
   replicas: 48

yarn-project/end-to-end/Earthfile

@@ -292,12 +292,12 @@ e2e-cli-wallet:
   LOCALLY
   RUN COMPOSE_FILE=scripts/docker-compose-wallet.yml ./scripts/e2e_compose_test.sh e2e_cli_wallet
 
-network-smoke:
+kind-network-smoke:
   ARG values_file
   LOCALLY
   RUN NAMESPACE=smoke FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/smoke.test.ts
 
-network-transfer:
+kind-network-transfer:
   ARG values_file
   LOCALLY
   RUN NAMESPACE=transfer FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/transfer.test.ts

yarn-project/end-to-end/scripts/network_test.sh

@@ -16,6 +16,13 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Main positional parameter
 TEST="$1"
 
+REPO=$(git rev-parse --show-toplevel)
+if [ "$(uname)" = "Linux" ] && [ "$(uname -m)" = "x86_64" ]; then
+  "$REPO"/spartan/scripts/setup_local_k8s.sh
+else
+  echo "Not on x64 Linux, not installing k8s and helm."
+fi
+
 # Default values for environment variables
 VALUES_FILE="${VALUES_FILE:-default.yaml}"
 CHAOS_VALUES="${CHAOS_VALUES:-}"
@@ -55,12 +62,13 @@ function show_status_until_pxe_ready() {
 }
 
 show_status_until_pxe_ready &
+SHOW_STATUS_PID=$!
 
 # Install the Helm chart
-helm upgrade --install spartan "$(git rev-parse --show-toplevel)/spartan/aztec-network/" \
+helm upgrade --install spartan "$REPO/spartan/aztec-network/" \
       --namespace "$NAMESPACE" \
       --create-namespace \
-      --values "$(git rev-parse --show-toplevel)/spartan/aztec-network/values/$VALUES_FILE" \
+      --values "$REPO/spartan/aztec-network/values/$VALUES_FILE" \
       --set images.aztec.image="aztecprotocol/aztec:$AZTEC_DOCKER_TAG" \
       --set ingress.enabled=true \
       --wait \
@@ -71,8 +79,16 @@ kubectl wait pod -l app==pxe --for=condition=Ready -n "$NAMESPACE" --timeout=10m
 
 # tunnel in to get access directly to our PXE service in k8s
 (kubectl port-forward --namespace $NAMESPACE svc/spartan-aztec-network-pxe 9082:8080 2>/dev/null >/dev/null || true) &
+PORT_FORWARD_PID=$!
+
+cleanup() {
+  echo "Cleaning up..."
+  kill $PORT_FORWARD_PID || true
+  kill $SHOW_STATUS_PID || true
+}
+
+trap cleanup EXIT SIGINT SIGTERM
 
-# run our test in the host network namespace (so we can access the above with localhost)
 docker run --rm --network=host \
   -e PXE_URL=http://localhost:9082 \
   -e DEBUG="aztec:*" \