From b6fee1a7227c735ff1eb3c1737e2416972bcf134 Mon Sep 17 00:00:00 2001 From: Alessandro De Blasis Date: Wed, 18 Jan 2023 13:58:17 +0000 Subject: [PATCH 1/6] fix(persistence): manual GC on ResetToGenesis --- persistence/debug.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/persistence/debug.go b/persistence/debug.go index 90ade24ae..cc5676c1b 100644 --- a/persistence/debug.go +++ b/persistence/debug.go @@ -3,6 +3,7 @@ package persistence import ( "crypto/sha256" "log" + "runtime/debug" "github.com/celestiaorg/smt" "github.com/pokt-network/pocket/persistence/types" @@ -88,6 +89,8 @@ func (m *persistenceModule) clearAllState(_ *messaging.DebugMessage) error { } log.Println("Cleared all the state") + // reclaming memory manually because the above calls deallocate and reallocate a lot of memory + debug.FreeOSMemory() return nil } From 7d8503d0c24915294059cc7fb53a355b33f411df Mon Sep 17 00:00:00 2001 From: Alessandro De Blasis Date: Wed, 18 Jan 2023 14:16:17 +0000 Subject: [PATCH 2/6] docs(persistence): CHANGELOG --- persistence/docs/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/persistence/docs/CHANGELOG.md b/persistence/docs/CHANGELOG.md index 9a341e225..5e367a1b7 100644 --- a/persistence/docs/CHANGELOG.md +++ b/persistence/docs/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.23] - 2023-01-18 + +- Added `debug.FreeOSMemory()` on `ResetToGenesis` to free-up memory and stabilize `LocalNet`. + ## [0.0.0.22] - 2023-01-14 - Add `max_conns_count`, `min_conns_count`, `max_conn_lifetime`, `max_conn_idle_time` and `health_check_period` to `PersistenceConfig`. From 5af31a92639ba533e903890697334bf0661fc55b Mon Sep 17 00:00:00 2001 From: Alessandro De Blasis Date: Mon, 23 Jan 2023 07:59:41 +0000 Subject: [PATCH 3/6] feat(RPC): feature-flagged pprof endpoint --- build/scripts/watch.sh | 7 ++++++- rpc/pprof.go | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 rpc/pprof.go diff --git a/build/scripts/watch.sh b/build/scripts/watch.sh index 157b1294f..26fe10a25 100755 --- a/build/scripts/watch.sh +++ b/build/scripts/watch.sh @@ -3,11 +3,16 @@ config=$1 genesis=$2 +tags="" +if [ -n "$PPROF_ENABLED" ] && [ "$PPROF_ENABLED" -eq "true" ]; then + tags="-tags=pprof_enabled" +fi + # TODO(olshansky): Make sure this works on everyone's workstations with Docker & Goland debugging. if [ -z "$DEBUG_PORT" ]; then echo "DEBUG DISABLED" - command="go run app/pocket/main.go --config=$config --genesis=$genesis" + command="go run $tags app/pocket/main.go --config=$config --genesis=$genesis" else echo "DEBUG ENABLED on port $DEBUG_PORT" command="touch /tmp/output.dlv && dlv debug app/pocket/main.go --headless --accept-multiclient --listen=:$DEBUG_PORT --api-version=2 --continue --output /tmp/output.dlv -- --config=$config --genesis=$genesis" diff --git a/rpc/pprof.go b/rpc/pprof.go new file mode 100644 index 000000000..f61bb1dcc --- /dev/null +++ b/rpc/pprof.go @@ -0,0 +1,19 @@ +//go:build pprof_enabled +// +build pprof_enabled + +package rpc + +import ( + "log" + "net/http" + _ "net/http/pprof" +) + +func init() { + go exposePProfEndpoint() +} + +func exposePProfEndpoint() { + log.Println("PProf endpoint exposed at localhost:6060") + log.Println(http.ListenAndServe("localhost:6060", nil)) +} From a315494b698a2960802e4e3be791c6a7f481b71d Mon Sep 17 00:00:00 2001 From: Alessandro De Blasis Date: Mon, 23 Jan 2023 13:13:48 +0000 Subject: [PATCH 4/6] docs(docs): profiling --- build/deployments/docker-compose.yaml | 2 + docs/development/PROFILING.md | 57 +++++++++++++++++++++++++++ docs/development/README.md | 6 +++ 3 files changed, 65 insertions(+) create mode 100644 docs/development/PROFILING.md diff --git a/build/deployments/docker-compose.yaml b/build/deployments/docker-compose.yaml index c9db5daf2..750043f2a 100755 --- a/build/deployments/docker-compose.yaml +++ b/build/deployments/docker-compose.yaml @@ -48,6 +48,8 @@ services: - "seccomp:unconfined" environment: - POCKET_RPC_USE_CORS=true + # Uncomment to enable the pprof server + # - PPROF_ENABLED=true # Uncomment to enable DLV debugging # - DEBUG_PORT=7081 diff --git a/docs/development/PROFILING.md b/docs/development/PROFILING.md new file mode 100644 index 000000000..1869c3842 --- /dev/null +++ b/docs/development/PROFILING.md @@ -0,0 +1,57 @@ +# Profiling + +This guide is not meant to replace any official documentation, but rather to provide a quick overview of the steps required to profile the node. + +## Profiling endpoint feature flag + +We pre-configured the node to expose a web server on port `6060` for the collection of profiling data. + +Because of the performance overhead, it is disabled by default. +In order to enable it, you can simply add the environment variable `PPROF_ENABLED=true` to the node. + +You can do that in several ways, the most obvious spot would be the `docker-compose.yml` file, in the `environment` section of the `node1.consensus` service. + +```yaml + environment: + # Uncomment to enable the pprof server + # - PPROF_ENABLED=true +``` + +### Example + +Let's assume that you want to profile the memory usage of the node during the execution of the `ResetToGenesis` command. + +These are the necessary steps: + +1. Enable the pprof server in the `docker-compose.yml` file (see above) +2. Collect a "baseline" heap profile of the node by running the following command: + +```bash +$ curl http://localhost:6060/debug/pprof/heap > baseline.heap +``` + +3. Run the `ResetToGenesis` command from the debug CLI +4. Collect another heap profile of the node by running the following command: + +```bash +curl http://localhost:6060/debug/pprof/heap > after_reset.heap +``` +5. Compare the two profiles using the `pprof` tool: + +```bash +$ go tool pprof -base=baseline.heap after_reset.heap +``` + +6. From the `pprof` prompt, you can run the `top` command to see the top 10 (can be any number) memory consumers: + +```bash +(pprof) top10 +``` + +7. By repeating the steps above, and comparing different heap profiles, you can identify the memory consumers that are causing the memory usage to grow. It could be a memory leak, or just a normal behavior. + +### Further reading + +- PProf: https://go.dev/blog/pprof +- Memory leaking scenarios: https://go101.org/article/memory-leaking.html + diff --git a/docs/development/README.md b/docs/development/README.md index 5f0cfb8cd..cf0017289 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -10,6 +10,7 @@ Please note that this repository is under very active development and breaking c - [View Available Commands](#view-available-commands) - [Running Unit Tests](#running-unit-tests) - [Running LocalNet](#running-localnet) + - [Profiling](#profiling) - [Code Organization](#code-organization) - [Linters](#linters) - [Installation of golangci-lint](#installation-of-golangci-lint) @@ -182,6 +183,11 @@ $ make client_start && make client_connect ✔ TriggerNextView # Let it rip! ``` +### Profiling + +If you need to profile the node for CPU and/or memory usage, you can use the `pprof` tool. +A quick guide is available [here](./PROFILING.md). + ## Code Organization ```bash From 7d17f6b63acb807c9ac688669ae296336ec912d8 Mon Sep 17 00:00:00 2001 From: Alessandro De Blasis Date: Mon, 23 Jan 2023 13:19:21 +0000 Subject: [PATCH 5/6] docs(shared): CHANGELOGs --- build/docs/CHANGELOG.md | 4 ++++ rpc/doc/CHANGELOG.md | 4 ++++ runtime/docs/CHANGELOG.md | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/build/docs/CHANGELOG.md b/build/docs/CHANGELOG.md index cd187114e..b7a594e7e 100644 --- a/build/docs/CHANGELOG.md +++ b/build/docs/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.6] - 2023-01-23 + +- Added pprof feature flag guideline in docker-compose.yml + ## [0.0.0.5] - 2023-01-20 - Update the docker-compose and relevant files to automatically load `pgadmin` server configs by binding the appropriate configs diff --git a/rpc/doc/CHANGELOG.md b/rpc/doc/CHANGELOG.md index f0a18ec47..7cac8ca97 100644 --- a/rpc/doc/CHANGELOG.md +++ b/rpc/doc/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.6] - 2023-01-23 + +- Added `pprof` http server feature flag via build tags + ## [0.0.0.5] - 2023-01-10 - Updated module constructor to accept a `bus` and not a `runtimeMgr` anymore diff --git a/runtime/docs/CHANGELOG.md b/runtime/docs/CHANGELOG.md index 0d0dd6fb3..defd03ec0 100644 --- a/runtime/docs/CHANGELOG.md +++ b/runtime/docs/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.9] - 2023-01-23 + +- Updated README.md with information about node profiling + ## [0.0.0.8] - 2023-01-19 - Rewrite `interface{}` to `any` From 3b5cfac160051177f35fd8d31c0b4476c555fd5e Mon Sep 17 00:00:00 2001 From: Alessandro De Blasis Date: Tue, 24 Jan 2023 17:59:41 +0000 Subject: [PATCH 6/6] Update persistence/docs/CHANGELOG.md Co-authored-by: Daniel Olshansky --- persistence/docs/CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/persistence/docs/CHANGELOG.md b/persistence/docs/CHANGELOG.md index 6aef482a4..05d2344b2 100644 --- a/persistence/docs/CHANGELOG.md +++ b/persistence/docs/CHANGELOG.md @@ -22,7 +22,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.0.0.23] - 2023-01-18 -- Consolidate the `Block` proto structures under `shared/core/types` - Remove `Block` proto definition to consolidate under `shared/core/types` ## [0.0.0.22] - 2023-01-14