monitoring dashboard

cmd/algorand-indexer/daemon.go

@@ -213,8 +213,14 @@ func handleBlock(block *rpcs.EncodedBlockCert, imp importer.Importer) error {
 	// Ignore round 0 (which is empty).
 	if block.Block.Round() > 0 {
 		metrics.BlockImportTimeSeconds.Observe(dt.Seconds())
-		metrics.ImportedTxnsPerBlock.Observe(float64(len(block.Block.Payset)))
 		metrics.ImportedRoundGauge.Set(float64(block.Block.Round()))
+		txnCountByType := make(map[string]int)
+		for _, txn := range block.Block.Payset {
+			txnCountByType[string(txn.Txn.Type)]++
+		}
+		for k, v := range txnCountByType {
+			metrics.ImportedTxnsPerBlock.WithLabelValues(k).Set(float64(v))
+		}
 	}
 
 	logger.Infof("round r=%d (%d txn) imported in %s", block.Block.Round(), len(block.Block.Payset), dt.String())

fetcher/fetcher.go

@@ -13,6 +13,7 @@ import (
 	"github.com/algorand/go-algorand-sdk/client/v2/algod"
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/rpcs"
+	"github.com/algorand/indexer/util/metrics"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -111,7 +112,13 @@ func (bot *fetcherImpl) catchupLoop(ctx context.Context) error {
 	var blockbytes []byte
 	aclient := bot.Algod()
 	for {
+		start := time.Now()
+
 		blockbytes, err = aclient.BlockRaw(bot.nextRound).Do(ctx)
+
+		dt := time.Since(start)
+		metrics.GetAlgodRawBlockTimeSeconds.Observe(dt.Seconds())
+
 		if err != nil {
 			// If context has expired.
 			if ctx.Err() != nil {
@@ -150,7 +157,13 @@ func (bot *fetcherImpl) followLoop(ctx context.Context) error {
 					"r=%d error getting status %d", retries, bot.nextRound)
 				continue
 			}
+			start := time.Now()
+
 			blockbytes, err = aclient.BlockRaw(bot.nextRound).Do(ctx)
+
+			dt := time.Since(start)
+			metrics.GetAlgodRawBlockTimeSeconds.Observe(dt.Seconds())
+
 			if err == nil {
 				break
 			} else if ctx.Err() != nil { // if context has expired

monitoring/Dockerfile-indexer

@@ -0,0 +1,33 @@
+ARG GO_VERSION=1.17.5
+FROM golang:$GO_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get install -y apt-utils curl git git-core bsdmainutils python3
+
+
+# Build go-algorand
+RUN mkdir /work
+WORKDIR /work
+ADD . ./
+WORKDIR /work/third_party/go-algorand
+RUN ./scripts/configure_dev.sh
+RUN make
+
+# Build indexer
+WORKDIR /work
+RUN make
+WORKDIR /work/cmd/algorand-indexer
+ENV CGO_ENABLED="1"
+RUN go build
+
+# The sleep is to wait until postgres starts
+CMD ["/bin/sh", "-c", "\
+  echo $ALGOD_NET && \
+  echo $CONNECTION_STRING &&\
+  sleep 5 && \
+  ./algorand-indexer daemon \
+    --server \":${PORT}\" \
+    -P \"${CONNECTION_STRING}\" \
+    --metrics-mode VERBOSE \
+    --algod-net \"${ALGOD_NET}\" \
+    --algod-token ${ALGOD_TOKEN}"]

monitoring/README.md

@@ -0,0 +1,60 @@
+## Indexer Monitoring Dashboard
+A monitoring dashboard displaying indexer performance metrics. 
+
+### To start a monitoring dashboard
+### Prerequisite
+ - Algorand indexer is running with metrics-mode set to ON or VERBOSE 
+
+Data sources for Grafana are set in `grafana_prometheus_datasource.yml`. Check that configurations for
+Prometheus source and PostgresSQL source are correct or Grafana will not have the metrics.
+
+Also, Prometheus target should be updated to listen to a correct indexer address (host:port). The default target assumes
+the indexer is running on prometheus container's host at port 8888. Check `<prometheus-url>/targets` to see whether Prometheus
+connects to the target successfully. 
+
+```json
+static_configs:
+- targets: ['host.docker.internal:8888'] 
+```
+
+Run,
+
+`docker-compose up -d prometheus grafana` 
+
+- Check metrics are written to <indexer-url>/metrics 
+- Grafana is running on http://localhost:3000; default login (admin/admin)
+
+
+
+### View metrics on grafana
+
+- Go to Import and upload dashboard.json
+- Run `create extension pg_stat_statements;` sql on db to enable query stats from Postgres. 
+  - If getting error such as `elation "pg_stat_statements" does not exist`, check `pg_stat_statements` is 
+   loaded to shared_preload_libraries in postgresql.conf or added in db start up command `-c shared_preload_libraries=pg_stat_statements`. 
+
+![](examples/widgets.png)
+
+
+**Default widgets**:
+
+Importing
+- Block import time
+- Transactions per block by type
+- TPS
+- Round #
+
+Query
+
+- Request duration
+- Request duraton by url
+- Postgres Eval Time
+
+System
+
+- average CPU usage
+- average Heap Alloc
+- Postgres table info
+- Postgres queries info
+
+