Merge pull request #1033 from permaweb/feat/fix-image-pipelines

feat/fix-image-pipelines

.github/workflows/devnet-ecr-push.yml

@@ -0,0 +1,135 @@
+name: 🏗️  Build and Push Docker images to devnet ECR
+
+on:
+  pull_request:
+    branches:
+        - '*'
+
+jobs:  
+  build-and-push-cu:
+    runs-on: ubuntu-latest
+    env:
+      REPOSITORY: devnet-ecr
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Extract shortened commit hash
+        id: vars
+        run: echo "COMMIT_HASH=$(echo $GITHUB_SHA | cut -c1-7)" >> $GITHUB_ENV
+
+      - name: Log in to Amazon ECR
+        id: login-devnet-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+        with:
+          mask-password: true
+          skip-logout: true
+          registry-type: private
+          registries: ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_AO_DEVNET }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_AO_DEVNET }}
+          AWS_REGION: ${{ secrets.AWS_REGION_AO_DEVNET }}
+
+      - name: Build and push CU Docker image
+        run: |
+          docker build -t ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-cu-$COMMIT_HASH -f servers/cu/Dockerfile servers/cu/
+          docker push ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-cu-$COMMIT_HASH
+
+  build-and-push-mu:
+    env:
+      REPOSITORY: devnet-ecr
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Extract shortened commit hash
+        id: vars
+        run: echo "COMMIT_HASH=$(echo $GITHUB_SHA | cut -c1-7)" >> $GITHUB_ENV
+
+      - name: Log in to Amazon ECR
+        id: login-devnet-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+        with:
+          mask-password: true
+          skip-logout: true
+          registry-type: private
+          registries: ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_AO_DEVNET }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_AO_DEVNET }}
+          AWS_REGION: ${{ secrets.AWS_REGION_AO_DEVNET }}
+
+      - name: Build and push MU Docker image
+        run: |
+          docker build -t ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-mu-$COMMIT_HASH -f servers/mu/Dockerfile servers/mu/
+          docker push ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-mu-$COMMIT_HASH
+
+  build-and-push-su:
+    env:
+      REPOSITORY: devnet-ecr
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Extract shortened commit hash
+        id: vars
+        run: echo "COMMIT_HASH=$(echo $GITHUB_SHA | cut -c1-7)" >> $GITHUB_ENV
+
+      - name: Log in to Amazon ECR
+        id: login-devnet-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+        with:
+          mask-password: true
+          skip-logout: true
+          registry-type: private
+          registries: ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_AO_DEVNET }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_AO_DEVNET }}
+          AWS_REGION: ${{ secrets.AWS_REGION_AO_DEVNET }}
+
+      - name: Build and push SU Docker image
+        run: |
+          docker build -t ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-su-$COMMIT_HASH -f servers/su/Dockerfile.x86 servers/su/
+          docker push ${{ secrets.AWS_ACCOUNT_ID_AO_DEVNET }}.dkr.ecr.${{ secrets.AWS_REGION_AO_DEVNET }}.amazonaws.com/$REPOSITORY:ao-su-$COMMIT_HASH
+
+  update-devnet-image-tags:
+      runs-on: ubuntu-latest
+      needs: 
+        - build-and-push-cu
+        - build-and-push-mu
+        - build-and-push-su
+
+      steps:
+        - name: Checkout Helmfile Repository
+          run: |
+            git clone https://$GITHUB_USERNAME:${{ secrets.AO_TO_CHARTS_PAT }}@github.com/permaweb/charts.git
+          env:
+            GITHUB_USERNAME: bredamatt
+
+        - name: Extract shortened commit hash
+          id: vars
+          run: echo "COMMIT_HASH=$(echo $GITHUB_SHA | cut -c1-7)" >> $GITHUB_ENV
+
+        - name: Update commit-sha.yaml
+          env:
+            TRUNCATED_SHA: ${{ env.COMMIT_HASH }}
+          run: |
+            cd charts
+            sed -i "s/commitSHA: \".*\"/commitSHA: \"${TRUNCATED_SHA}\"/" helmfile/devnet/commit-sha.yaml
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git commit -am "fix: from ao-repo - update commit-sha to ${TRUNCATED_SHA} for syncing latest image tags"
+            git push https://${{ secrets.AO_TO_CHARTS_PAT }}@github.com/permaweb/charts.git main

.github/workflows/su.yml

@@ -18,20 +18,20 @@ defaults:
 
 jobs:
 
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v4
+  # test:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: ⬇️ Checkout repo
+  #       uses: actions/checkout@v4
 
-      - name: ⎔ Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
+  #     - name: ⎔ Setup Rust
+  #       uses: actions-rs/toolchain@v1
+  #       with:
+  #         toolchain: 1.75.0
 
-      - name: Run Tests
-        working-directory: servers/su
-        run: cargo test
+  #     - name: Run Tests
+  #       working-directory: servers/su
+  #       run: cargo test
 
 
   publish:

servers/cu/Dockerfile

@@ -6,7 +6,9 @@ COPY ./package.json .
 COPY ./package-lock.json .
 COPY ./src ./src
 
-RUN npm install --omit=dev
+RUN npm install --omit=dev && \
+    mkdir /db/ && \
+    mkdir /file-checkpoints/
 
 ENV NODE_ENV=production
 

servers/mu/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:21
+FROM node:22
 
 RUN apt update && apt install -y bash git awscli
 
@@ -7,14 +7,11 @@ WORKDIR /usr/app
 COPY ./package.json .
 COPY ./package-lock.json .
 COPY ./src ./src
-COPY ./entrypoint.sh ./entrypoint.sh
 
-RUN chmod +x /usr/app/entrypoint.sh
+RUN mkdir /usr/app/mu-data/
 
 RUN npm install --ignore-engines
 
-ENV NODE_ENV=production
-
 EXPOSE 3005
 
-ENTRYPOINT ["/usr/app/entrypoint.sh"]
+CMD ["node", "src/app.js"]

servers/su/Dockerfile

@@ -49,4 +49,4 @@ LABEL build_instructions="To build just the binary, run the following command: d
 ENTRYPOINT [ "/app/su" ]
 
 # Run time command arguments, default is empty
-CMD []
+CMD []

servers/su/Dockerfile.arm64

@@ -0,0 +1,40 @@
+### NOTE: THIS WON'T WORK ON AN X86 DEVICE
+
+# Stage 1: Planner
+FROM --platform=linux/arm64  rust:1.75.0 AS planner
+WORKDIR /app
+RUN cargo install cargo-chef
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# Stage 2: Cache the build of the dependencies
+FROM --platform=linux/arm64 rust:1.75.0 AS cacher
+WORKDIR /app
+RUN cargo install cargo-chef
+RUN apt-get update && apt-get install -y \
+    llvm-dev \
+    libclang-dev \
+    clang \
+    librocksdb-dev \
+    libpq5 \
+    libssl-dev
+COPY --from=planner /app/recipe.json recipe.json
+RUN ls /lib/
+RUN cargo chef cook --release --recipe-path recipe.json
+
+# Stage 3: Build binary with pre-built and cached dependencies
+FROM --platform=linux/arm64 rust:1.75.0 AS builder
+COPY . /app
+WORKDIR /app
+COPY --from=cacher /app/target target
+COPY --from=cacher /usr/local/cargo /usr/local/cargo
+COPY --from=cacher /lib/aarch64-linux-gnu/*  /lib/aarch64-linux-gnu/
+# Set the correct Rust target based on architecture
+RUN cargo build --release
+
+# Stage 4: Runner 
+FROM --platform=linux/arm64 gcr.io/distroless/cc-debian12 
+RUN apt-get update && apt install ca-certificates openssl -y
+COPY --from=builder /app/target/release/su /
+COPY --from=cacher /lib/aarch64-linux-gnu/*  /lib/aarch64-linux-gnu/
+CMD ["sh", "-c", "sleep 10 && ./su su 9000"]

servers/su/Dockerfile.x86

@@ -0,0 +1,41 @@
+### NOTE: THIS WON'T WORK ON AN ARM64 DEVICE
+
+# Stage 1: Planner
+FROM --platform=linux/amd64 rust:1.75.0 AS planner
+WORKDIR /app
+RUN cargo install cargo-chef
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# Stage 2: Cache the build of the dependencies
+FROM --platform=linux/amd64 rust:1.75.0 AS cacher
+WORKDIR /app
+RUN cargo install cargo-chef
+RUN apt-get update && apt-get install -y \
+    llvm-dev \
+    libclang-dev \
+    clang \
+    librocksdb-dev \
+    libpq5 \
+    libssl-dev 
+
+COPY --from=planner /app/recipe.json recipe.json
+RUN ls /lib/
+RUN cargo chef cook --release --recipe-path recipe.json
+
+# Stage 3: Build binary with pre-built and cached dependencies
+FROM --platform=linux/amd64 rust:1.75.0 AS builder
+COPY . /app
+WORKDIR /app
+COPY --from=cacher /app/target target
+COPY --from=cacher /usr/local/cargo /usr/local/cargo
+COPY --from=cacher /lib/x86_64-linux-gnu/* /lib/x86_64-linux-gnu/
+# Set the correct Rust target based on architecture
+RUN cargo build --release
+
+# Stage 4: Runner 
+FROM --platform=linux/amd64 ubuntu:22.04
+RUN apt-get update && apt install ca-certificates openssl -y
+COPY --from=builder /app/target/release/su /
+COPY --from=cacher /lib/x86_64-linux-gnu/*  /lib/x86_64-linux-gnu/
+CMD ["sh", "-c", "sleep 10 && ./su su 9000"]