From e4d33e4488e1793eaccf4c654cec1e6db4e08a21 Mon Sep 17 00:00:00 2001 From: birdy Date: Mon, 24 Apr 2023 11:33:42 -0700 Subject: [PATCH 1/2] feat: add grafana agent and refactor configs --- deployment/{lily/lily.env => .env.example} | 18 +++++++++++++--- deployment/README.md | 17 +++++++++------ deployment/agent/config.yml | 15 +++++++++++++ deployment/docker-compose-notifier.yml | 25 ++++++++++++++++++---- deployment/docker-compose-worker.yml | 25 ++++++++++++++++++---- deployment/lily/config.toml | 2 +- deployment/lily/notifier_docker_init.sh | 2 +- deployment/lily/worker_docker_init.sh | 2 +- deployment/promtail/config.yml | 4 ++-- deployment/promtail/promtail.env | 2 -- 10 files changed, 87 insertions(+), 25 deletions(-) rename deployment/{lily/lily.env => .env.example} (63%) create mode 100644 deployment/agent/config.yml delete mode 100644 deployment/promtail/promtail.env diff --git a/deployment/lily/lily.env b/deployment/.env.example similarity index 63% rename from deployment/lily/lily.env rename to deployment/.env.example index 7e105c47a..64d265e56 100644 --- a/deployment/lily/lily.env +++ b/deployment/.env.example @@ -14,8 +14,8 @@ GOLOG_LOG_LEVEL=info LILY_LOG_LEVEL_NAMED=vm:error,badgerbs:error # Lily local cache size -LILY_BLOCKSTORE_CACHE_SIZE=1000000 -LILY_STATESTORE_CACHE_SIZE=1000000 +LILY_BLOCKSTORE_CACHE_SIZE=2000000 +LILY_STATESTORE_CACHE_SIZE=2000000 # Enable for vm_messages LOTUS_VM_ENABLE_TRACING=1 @@ -23,5 +23,17 @@ LOTUS_VM_ENABLE_TRACING=1 # Queue configuration LILY_REDIS_ADDR= LILY_REDIS_DB= -LILY_REDIS_USERNAME= +LILY_REDIS_USERNAME= LILY_REDIS_PASSWORD= + +# Promtail environment label +PROMTAIL_ENV= +# Promtail remote url +PROMTAIL_REMOTE_URL= + +# Tracing remote url +TRACING_REMOTE_URL= +# Tracing remote username +TRACING_REMOTE_USERNAME= +# Tracing remote password +TRACING_REMOTE_PASSWORD= \ No newline at end of file diff --git a/deployment/README.md b/deployment/README.md index f00b09d88..52e8df3ad 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -4,14 +4,17 @@ - example: [install in debian](https://docs.docker.com/engine/install/debian/) - git -## Provide the necessary URL and Password -- In **lily/lily.env** +## Provide the required credentials and configurations +- Rename **.env.example** to **.env** and provide the following env vars: - `LILY_REDIS_ADDR` + - `LILY_REDIS_USERNAME` - `LILY_REDIS_PASSWORD` - `LILY_STORAGE_POSTGRESQL_DB_URL` + - `PROMTAIL_ENV`: default value is `lily_gce_staging` + - `PROMTAIL_REMOTE_URL` + - `TRACING_REMOTE_URL` + - `TRACING_REMOTE_USERNAME` + - `TRACING_REMOTE_PASSWORD` - In **lily/config.toml** - - `[Storage.Postgresql.Database1]`: to replace the `POSTGRESQL_URL` - - `[Queue.Notifiers.Notifier1]`: to replace the `REDIS_ADDRESS` -- In **promtail/promtail.env** - - `REMOTE_LOKI_URL` - - `LILY_ENV`: default value is `lily_gce_staging` + - `[Queue.Notifiers.Notifier1]`: replace `REDIS_ADDRESS` + diff --git a/deployment/agent/config.yml b/deployment/agent/config.yml new file mode 100644 index 000000000..a03f8c89f --- /dev/null +++ b/deployment/agent/config.yml @@ -0,0 +1,15 @@ +traces: + configs: + - name: default + remote_write: + - endpoint: ${TRACING_REMOTE_URL} + basic_auth: + username: ${TRACING_REMOTE_USERNAME} + password: ${TRACING_REMOTE_PASSWORD} + receivers: + jaeger: + protocols: + grpc: + thrift_binary: + thrift_compact: + thrift_http: diff --git a/deployment/docker-compose-notifier.yml b/deployment/docker-compose-notifier.yml index ed1efe020..b23d9c835 100644 --- a/deployment/docker-compose-notifier.yml +++ b/deployment/docker-compose-notifier.yml @@ -7,12 +7,11 @@ services: notifier: image: filecoin/lily:v0.13.0 container_name: lily-notifier - env_file: - # Check envvars for configurable options - - ./lily/lily.env ports: # export the metrics port - 9991:9991 + env_file: + - ./.env volumes: # holds lily datastore repo - lily_notifier_data:/var/lib/lily @@ -52,7 +51,7 @@ services: image: grafana/promtail:2.7.0 container_name: promtail env_file: - - ./promtail/promtail.env + - ./.env volumes: - ./promtail/config.yml:/etc/promtail/config.yml - /var/run/docker.sock:/var/run/docker.sock @@ -60,6 +59,24 @@ services: entrypoint: [ "/bin/sh","-c" ] command: - "/usr/bin/promtail -config.file=/etc/promtail/config.yml -client.external-labels=hostname=$(tail -2 /etc/hosts | cut -d' ' -f1 | head -1) -config.expand-env" + grafana-agent: + image: "grafana/agent:v0.32.1" + restart: always + pid: "host" + env_file: + - ./.env + volumes: + - ./agent:/etc/agent + entrypoint: + - /bin/agent + - -server.http.address=0.0.0.0:12345 + - -config.file=/etc/agent/config.yml + - -config.expand-env + - -config.enable-read-api + ports: + - "12345:12345" + networks: + - notifier-network volumes: lily_notifier_data: {} diff --git a/deployment/docker-compose-worker.yml b/deployment/docker-compose-worker.yml index 6c3e7b702..250ee4511 100644 --- a/deployment/docker-compose-worker.yml +++ b/deployment/docker-compose-worker.yml @@ -7,12 +7,11 @@ services: worker: image: filecoin/lily:v0.13.0 container_name: lily-worker - env_file: - # Check envvars for configurable options - - ./lily/lily.env ports: # export the metrics port - 9991:9991 + env_file: + - ./.env volumes: # holds lily datastore repo - lily_worker_data:/var/lib/lily @@ -52,7 +51,7 @@ services: image: grafana/promtail:2.7.0 container_name: promtail env_file: - - ./promtail/promtail.env + - ./.env volumes: - ./promtail/config.yml:/etc/promtail/config.yml - /var/run/docker.sock:/var/run/docker.sock @@ -60,6 +59,24 @@ services: entrypoint: [ "/bin/sh","-c" ] command: - "/usr/bin/promtail -config.file=/etc/promtail/config.yml -client.external-labels=hostname=$(tail -2 /etc/hosts | cut -d' ' -f1 | head -1) -config.expand-env" + grafana-agent: + image: "grafana/agent:v0.32.1" + restart: always + pid: "host" + env_file: + - ./.env + volumes: + - ./agent:/etc/agent + entrypoint: + - /bin/agent + - -server.http.address=0.0.0.0:12345 + - -config.file=/etc/agent/config.yml + - -config.expand-env + - -config.enable-read-api + ports: + - "12345:12345" + networks: + - worker-network volumes: lily_worker_data: {} diff --git a/deployment/lily/config.toml b/deployment/lily/config.toml index 3f28ea9e6..36d16f358 100644 --- a/deployment/lily/config.toml +++ b/deployment/lily/config.toml @@ -59,7 +59,7 @@ FilePattern = "{table}.csv" [Storage.Postgresql] [Storage.Postgresql.Database1] - URL = "POSTGRESQL_URL" + URLEnv = "LILY_STORAGE_POSTGRESQL_DB_URL" ApplicationName = "lily" SchemaName = "visor" PoolSize = 20 diff --git a/deployment/lily/notifier_docker_init.sh b/deployment/lily/notifier_docker_init.sh index e0208009a..88aba343d 100755 --- a/deployment/lily/notifier_docker_init.sh +++ b/deployment/lily/notifier_docker_init.sh @@ -27,7 +27,7 @@ fi chmod -R 0600 ${LILY_REPO}/keystore -lily daemon --repo=/var/lib/lily --config=/var/lib/lily/config.toml & +lily --jaeger-tracing --jaeger-provider-url=http://grafana-agent:14268/api/traces --jaeger-sampler-ratio=0.001 daemon --repo=/var/lib/lily --config=/var/lib/lily/config.toml & # wait for lily daemon sleep 10 diff --git a/deployment/lily/worker_docker_init.sh b/deployment/lily/worker_docker_init.sh index 188a05120..18669f514 100755 --- a/deployment/lily/worker_docker_init.sh +++ b/deployment/lily/worker_docker_init.sh @@ -27,7 +27,7 @@ fi chmod -R 0600 ${LILY_REPO}/keystore -lily daemon --repo=/var/lib/lily --config=/var/lib/lily/config.toml & +lily --jaeger-tracing --jaeger-provider-url=http://grafana-agent:14268/api/traces --jaeger-sampler-ratio=0.001 daemon --repo=/var/lib/lily --config=/var/lib/lily/config.toml & # wait for lily daemon sleep 10 diff --git a/deployment/promtail/config.yml b/deployment/promtail/config.yml index 04654245c..3f46d8409 100644 --- a/deployment/promtail/config.yml +++ b/deployment/promtail/config.yml @@ -6,7 +6,7 @@ positions: filename: /tmp/positions.yaml clients: - - url: ${REMOTE_LOKI_URL} + - url: ${PROMTAIL_REMOTE_URL} scrape_configs: - job_name: system @@ -29,4 +29,4 @@ scrape_configs: - docker: {} - static_labels: job: docker - environment: ${LILY_ENV:lily_gce_testing} + environment: ${PROMTAIL_ENV:lily_gce_testing} diff --git a/deployment/promtail/promtail.env b/deployment/promtail/promtail.env deleted file mode 100644 index 1f22bb3e7..000000000 --- a/deployment/promtail/promtail.env +++ /dev/null @@ -1,2 +0,0 @@ -LILY_ENV=lily_gce_staging -REMOTE_LOKI_URL= From 0370b8167c638267b5ad8b24139f5ecc11dc6a95 Mon Sep 17 00:00:00 2001 From: Terry Date: Tue, 25 Apr 2023 12:29:23 +0800 Subject: [PATCH 2/2] turn on the splitstore functionality --- deployment/lily/config.toml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/deployment/lily/config.toml b/deployment/lily/config.toml index 36d16f358..c577a683d 100644 --- a/deployment/lily/config.toml +++ b/deployment/lily/config.toml @@ -31,16 +31,10 @@ OffChainRetrieval = false [Chainstore] - EnableSplitstore = false + EnableSplitstore = true [Chainstore.Splitstore] - ColdStoreType = "" - HotStoreType = "" - MarkSetType = "" - HotStoreMessageRetention = 0 - HotStoreFullGCFrequency = 0 + ColdStoreType = "discard" EnableColdStoreAutoPrune = false - ColdStoreFullGCFrequency = 0 - ColdStoreRetention = 0 [Fevm] # EnableEthRPC enables eth_ rpc, and enables storing a mapping of eth transaction hashes to filecoin message Cids.