feat: add grafana agent and refactor configs

deployment/lily/lily.env → deployment/.env.example

@@ -14,14 +14,26 @@ GOLOG_LOG_LEVEL=info
 LILY_LOG_LEVEL_NAMED=vm:error,badgerbs:error
 
 # Lily local cache size
-LILY_BLOCKSTORE_CACHE_SIZE=1000000
-LILY_STATESTORE_CACHE_SIZE=1000000
+LILY_BLOCKSTORE_CACHE_SIZE=2000000
+LILY_STATESTORE_CACHE_SIZE=2000000
 
 # Enable for vm_messages
 LOTUS_VM_ENABLE_TRACING=1
 
 # Queue configuration
 LILY_REDIS_ADDR=
 LILY_REDIS_DB=
-LILY_REDIS_USERNAME= 
+LILY_REDIS_USERNAME=
 LILY_REDIS_PASSWORD=
+
+# Promtail environment label
+PROMTAIL_ENV=
+# Promtail remote url
+PROMTAIL_REMOTE_URL=
+
+# Tracing remote url
+TRACING_REMOTE_URL=
+# Tracing remote username
+TRACING_REMOTE_USERNAME=
+# Tracing remote password
+TRACING_REMOTE_PASSWORD=

deployment/README.md

@@ -4,14 +4,17 @@
   - example: [install in debian](https://docs.docker.com/engine/install/debian/)
 - git
 
-## Provide the necessary URL and Password
-- In **lily/lily.env**
+## Provide the required credentials and configurations
+- Rename **.env.example** to **.env** and provide the following env vars:
   - `LILY_REDIS_ADDR`
+  - `LILY_REDIS_USERNAME`
   - `LILY_REDIS_PASSWORD`
   - `LILY_STORAGE_POSTGRESQL_DB_URL`
+  - `PROMTAIL_ENV`: default value is `lily_gce_staging`
+  - `PROMTAIL_REMOTE_URL`
+  - `TRACING_REMOTE_URL`
+  - `TRACING_REMOTE_USERNAME`
+  - `TRACING_REMOTE_PASSWORD`
 - In **lily/config.toml**
-  - `[Storage.Postgresql.Database1]`: to replace the `POSTGRESQL_URL` 
-  - `[Queue.Notifiers.Notifier1]`: to replace the `REDIS_ADDRESS`
-- In **promtail/promtail.env**
-  - `REMOTE_LOKI_URL`
-  - `LILY_ENV`: default value is `lily_gce_staging`
+  - `[Queue.Notifiers.Notifier1]`: replace `REDIS_ADDRESS`
+

deployment/agent/config.yml

@@ -0,0 +1,15 @@
+traces:
+  configs:
+    - name: default
+      remote_write:
+        - endpoint: ${TRACING_REMOTE_URL}
+          basic_auth:
+            username: ${TRACING_REMOTE_USERNAME}
+            password: ${TRACING_REMOTE_PASSWORD}
+      receivers:
+        jaeger:
+          protocols:
+            grpc:
+            thrift_binary:
+            thrift_compact:
+            thrift_http:

deployment/docker-compose-notifier.yml

@@ -7,12 +7,11 @@ services:
   notifier:
     image: filecoin/lily:v0.13.0
     container_name: lily-notifier
-    env_file:
-      # Check envvars for configurable options
-      - ./lily/lily.env
     ports:
       # export the metrics port
       - 9991:9991
+    env_file:
+      - ./.env
     volumes:
       # holds lily datastore repo
       - lily_notifier_data:/var/lib/lily
@@ -52,14 +51,32 @@ services:
     image: grafana/promtail:2.7.0
     container_name: promtail
     env_file:
-      - ./promtail/promtail.env
+      - ./.env
     volumes:
       - ./promtail/config.yml:/etc/promtail/config.yml
       - /var/run/docker.sock:/var/run/docker.sock
       - /etc/hosts:/etc/hosts:ro
     entrypoint: [ "/bin/sh","-c" ]
     command:
       - "/usr/bin/promtail -config.file=/etc/promtail/config.yml -client.external-labels=hostname=$(tail -2 /etc/hosts | cut -d' ' -f1 | head -1)  -config.expand-env"
+  grafana-agent:
+    image: "grafana/agent:v0.32.1"
+    restart: always
+    pid: "host"
+    env_file:
+      - ./.env
+    volumes:
+      - ./agent:/etc/agent
+    entrypoint:
+      - /bin/agent
+      - -server.http.address=0.0.0.0:12345
+      - -config.file=/etc/agent/config.yml
+      - -config.expand-env
+      - -config.enable-read-api
+    ports:
+      - "12345:12345"
+    networks:
+      - notifier-network
 
 volumes:
   lily_notifier_data: {}

deployment/docker-compose-worker.yml

@@ -7,12 +7,11 @@ services:
   worker:
     image: filecoin/lily:v0.13.0
     container_name: lily-worker
-    env_file:
-      # Check envvars for configurable options
-      - ./lily/lily.env
     ports:
       # export the metrics port
       - 9991:9991
+    env_file:
+      - ./.env
     volumes:
       # holds lily datastore repo
       - lily_worker_data:/var/lib/lily
@@ -52,14 +51,32 @@ services:
     image: grafana/promtail:2.7.0
     container_name: promtail
     env_file:
-      - ./promtail/promtail.env
+      - ./.env
     volumes:
       - ./promtail/config.yml:/etc/promtail/config.yml
       - /var/run/docker.sock:/var/run/docker.sock
       - /etc/hosts:/etc/hosts:ro
     entrypoint: [ "/bin/sh","-c" ]
     command:
       - "/usr/bin/promtail -config.file=/etc/promtail/config.yml -client.external-labels=hostname=$(tail -2 /etc/hosts | cut -d' ' -f1 | head -1)  -config.expand-env"
+  grafana-agent:
+    image: "grafana/agent:v0.32.1"
+    restart: always
+    pid: "host"
+    env_file:
+      - ./.env
+    volumes:
+      - ./agent:/etc/agent
+    entrypoint:
+      - /bin/agent
+      - -server.http.address=0.0.0.0:12345
+      - -config.file=/etc/agent/config.yml
+      - -config.expand-env
+      - -config.enable-read-api
+    ports:
+      - "12345:12345"
+    networks:
+      - worker-network
 
 volumes:
   lily_worker_data: {}

deployment/lily/config.toml

@@ -31,16 +31,10 @@
   OffChainRetrieval = false
 
 [Chainstore]
-  EnableSplitstore = false
+  EnableSplitstore = true
   [Chainstore.Splitstore]
-    ColdStoreType = ""
-    HotStoreType = ""
-    MarkSetType = ""
-    HotStoreMessageRetention = 0
-    HotStoreFullGCFrequency = 0
+    ColdStoreType = "discard"
     EnableColdStoreAutoPrune = false
-    ColdStoreFullGCFrequency = 0
-    ColdStoreRetention = 0
 
 [Fevm]
   # EnableEthRPC enables eth_ rpc, and enables storing a mapping of eth transaction hashes to filecoin message Cids.
@@ -59,7 +53,7 @@
       FilePattern = "{table}.csv"
   [Storage.Postgresql]
     [Storage.Postgresql.Database1]
-      URL = "POSTGRESQL_URL"
+      URLEnv = "LILY_STORAGE_POSTGRESQL_DB_URL"
       ApplicationName = "lily"
       SchemaName = "visor"
       PoolSize = 20

deployment/lily/notifier_docker_init.sh

@@ -27,7 +27,7 @@ fi
 
 chmod -R 0600 ${LILY_REPO}/keystore
 
-lily daemon --repo=/var/lib/lily --config=/var/lib/lily/config.toml &
+lily --jaeger-tracing --jaeger-provider-url=http://grafana-agent:14268/api/traces --jaeger-sampler-ratio=0.001 daemon --repo=/var/lib/lily --config=/var/lib/lily/config.toml &
 
 # wait for lily daemon
 sleep 10

deployment/lily/worker_docker_init.sh

@@ -27,7 +27,7 @@ fi
 
 chmod -R 0600 ${LILY_REPO}/keystore
 
-lily daemon --repo=/var/lib/lily --config=/var/lib/lily/config.toml &
+lily --jaeger-tracing --jaeger-provider-url=http://grafana-agent:14268/api/traces --jaeger-sampler-ratio=0.001 daemon --repo=/var/lib/lily --config=/var/lib/lily/config.toml &
 
 # wait for lily daemon
 sleep 10

deployment/promtail/config.yml

@@ -6,7 +6,7 @@ positions:
   filename: /tmp/positions.yaml
 
 clients:
-  - url: ${REMOTE_LOKI_URL}
+  - url: ${PROMTAIL_REMOTE_URL}
 
 scrape_configs:
   - job_name: system
@@ -29,4 +29,4 @@ scrape_configs:
       - docker: {}
       - static_labels:
           job: docker
-          environment: ${LILY_ENV:lily_gce_testing}
+          environment: ${PROMTAIL_ENV:lily_gce_testing}