feat(blob-store): Add authentication

Signed-off-by: Diwank Singh Tomer <[email protected]>

.env.example

@@ -1,5 +1,14 @@
 # Security
 # --------
+
+# Please set these access keys to something random and unique.
+# Note: For just testing, you can set them to the same value.
+
+# On Linux, you can generate a random key with:
+# openssl rand -base64 32
+# OR
+# tr -dc 'A-Za-z0-9+_/' </dev/urandom | head -c 32; echo
+
 JWT_SHARED_KEY=<your_jwt_shared_key>
 AGENTS_API_KEY=<your_agents_api_key>
 COZO_AUTH_TOKEN=<your_cozo_auth_token>
@@ -37,6 +46,8 @@ LITELLM_REDIS_PASSWORD=<your_litellm_redis_password>
 # EMBEDDING_MODEL_ID=Alibaba-NLP/gte-large-en-v1.5
 # NUM_GPUS=1
 # INTEGRATION_SERVICE_URL=http://integrations:8000
+# USE_BLOB_STORE_FOR_TEMPORAL=false
+# BLOB_STORE_CUTOFF_KB=1024
 
 # Temporal
 # --------
@@ -69,3 +80,10 @@ LITELLM_REDIS_PASSWORD=<your_litellm_redis_password>
 # GITHUB_API_KEY=<your_github_api_key>
 # VOYAGE_API_KEY=<your_voyage_api_key>
 # GOOGLE_APPLICATION_CREDENTIALS=.keys/julep-vertexai-svc.json
+
+# Blob Store
+# -----------
+
+# S3_ENDPOINT=http://seaweedfs:8333
+# S3_ACCESS_KEY=<your_s3_access_key>
+# S3_SECRET_KEY=<your_s3_secret_key>

blob-store/.gitignore

@@ -0,0 +1 @@
+/s3.json

blob-store/Dockerfile

@@ -0,0 +1,23 @@
+# syntax=docker/dockerfile:1
+# check=error=true
+
+FROM chrislusf/seaweedfs
+
+# Install envsubst
+ENV BUILD_DEPS="gettext"  \
+    RUNTIME_DEPS="libintl"
+
+RUN set -x && \
+    apk add --update $RUNTIME_DEPS && \
+    apk add --virtual build_deps $BUILD_DEPS &&  \
+    cp /usr/bin/envsubst /usr/local/bin/envsubst && \
+    apk del build_deps
+
+# Expected environment variables:
+# - S3_ACCESS_KEY
+# - S3_SECRET_KEY
+
+COPY s3.json.template /s3.json.template
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

blob-store/docker-compose.yml

@@ -1,14 +1,19 @@
 name: julep-blob-store
 
-x-seaweedfs-base:
-  &seaweedfs-base
-  image: chrislusf/seaweedfs
-  profiles:
-    - blob-store
-
 services:
   seaweedfs:
-    <<: *seaweedfs-base
+    image: julepai/blob-store:${TAG}
+    build:
+      context: .
+      dockerfile: Dockerfile
+    profiles:
+      - blob-store
+
+    environment:
+      - S3_ACCESS_KEY=${S3_ACCESS_KEY}
+      - S3_SECRET_KEY=${S3_SECRET_KEY}
+      - DEBUG=${DEBUG:-true}
+
     ports:
       - 9333:9333  # master port
       - 8333:8333  # s3 port
@@ -17,7 +22,7 @@ services:
       # - 19333:19333  # master grpc port
       # - 18081:18080  # volume grpc port
       # - 18888:18888  # filer grpc port
-    command: "server -filer -s3 -dir=/data -ip=seaweedfs -ip.bind=0.0.0.0 -metricsPort=9321 -master.raftBootstrap=false -master.port=9333 -master.resumeState=true -volume.port=28080 -volume.index=leveldb -filer.port=8888 -s3.port=8333"
+    command: "-filer -s3 -dir=/data -ip=seaweedfs -ip.bind=0.0.0.0 -metricsPort=9321 -master.raftBootstrap=false -master.port=9333 -master.resumeState=true -volume.port=28080 -volume.index=leveldb -filer.port=8888 -s3.port=8333"
     healthcheck:
       test: [ "CMD", "wget", "-qSO", "-", "http://0.0.0.0:9333/cluster/healthz" ]
       interval: 60s

blob-store/entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -e
+
+# Check the environment variables
+for var_name in S3_ACCESS_KEY S3_SECRET_KEY
+do
+    if [ -z "`eval echo \\\$$var_name`" ]; then
+        echo "Error: Environment variable '$var_name' is not set."
+        exit 1
+    fi
+done
+
+# Generate the s3.json configuration file
+envsubst < /s3.json.template > /s3.json
+
+if [ "$DEBUG" = "true" ]; then
+    echo '@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@'
+    echo '@@@ Careful: Debug mode is enabled. @@@'
+    echo '@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@'
+
+    echo 'Printing s3.json:'
+    cat /s3.json
+fi
+
+# Forward all arguments to the seaweedfs binary
+exec weed server -s3.config=/s3.json "$@"

blob-store/s3.json.template

@@ -0,0 +1,33 @@
+{
+  "identities": [
+    {
+      "name": "anonymous",
+      "actions": [
+        "Read"
+      ]
+    },
+    {
+      "name": "julep",
+      "credentials": [
+        {
+          "accessKey": "${S3_ACCESS_KEY}",
+          "secretKey": "${S3_SECRET_KEY}"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    }
+  ],
+  "accounts": [
+      {
+        "id" : "julep",
+        "displayName": "Julep",
+        "emailAddress": "[email protected]"
+      }
+  ]
+}