feat(halo): ready command integrated into docker containers (#2179)

- Adds a new halo command `ready` indicating the readiness of the halo
node.
- Integrates the new command into the halo and halovisor images.
- Implements a new prometheus metrics exposing the readiness status

issue: #2091

cli/cmd/compose.yml.tpl

@@ -27,6 +27,10 @@ services:
       #- --metrics                    # Enable prometheus metrics
       #- --pprof                      # Enable prometheus metrics
       #- --pprof.addr=0.0.0.0         # Enable prometheus metrics
+    healthcheck:
+      test: "nc -z localhost 8545"
+      interval: 1s
+      retries: 30
 
     ports:
       - 8551             # Auth-RPC (used by halo)

halo/Dockerfile

@@ -8,6 +8,8 @@ COPY --from=alpine:latest /tmp /tmp
 
 COPY halo /app
 
+HEALTHCHECK CMD ["/app", "ready"]
+
 # Mount home directory at /halo
 VOLUME ["/halo"]
 

halo/app/metrics.go

@@ -47,6 +47,13 @@ var (
 		Name:      "size_bytes",
 		Help:      "Current size of the database directory in bytes.",
 	})
+
+	nodeReadiness = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: "halo",
+		Subsystem: "health",
+		Name:      "ready",
+		Help:      "Constant gauge of 1 if local halo node is ready, 0 if not.",
+	})
 )
 
 // setConstantGauge sets the value of a gauge to 1 if b is true, 0 otherwise.

halo/app/monitor.go

@@ -160,6 +160,25 @@ func monitorEVMOnce(ctx context.Context, ethCl ethclient.Client, status *readine
 	return nil
 }
 
+// instrumentReadiness exports the node readiness to prometheus.
+func instrumentReadiness(ctx context.Context, status *readinessStatus) {
+	ticker := time.NewTicker(time.Second * 10)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			var health float64
+			if status.ready() {
+				health = 1.0
+			}
+			nodeReadiness.Set(health)
+		}
+	}
+}
+
 // dirSize returns the total size of the directory at path.
 func dirSize(path string) (int64, error) {
 	var size int64

halo/app/start.go

@@ -205,6 +205,7 @@ func Start(ctx context.Context, cfg Config) (<-chan error, func(context.Context)
 	}
 
 	status := new(readinessStatus)
+	go instrumentReadiness(ctx, status)
 
 	stopMonitoringAPI := startMonitoringAPI(&cfg.Comet, asyncAbort, status)
 

halo/cmd/cmd.go

@@ -28,6 +28,7 @@ func New() *cobra.Command {
 		buildinfo.NewVersionCmd(),
 		newConsKeyCmd(),
 		newStatusCmd(),
+		newReadyCmd(),
 	)
 }
 

halo/cmd/flags.go

@@ -60,3 +60,9 @@ func bindStatusFlags(cmd *cobra.Command, cfg *statusConfig) {
 	flags.StringVarP(&cfg.Node, "node", "n", cfg.Node, "Node to connect to")
 	flags.StringVarP(&cfg.Output, "output", "o", cfg.Output, "Output format (text|json)")
 }
+
+func bindReadyFlags(cmd *cobra.Command, cfg *readyConfig) {
+	flags := cmd.Flags()
+
+	flags.StringVarP(&cfg.MonitoringURL, "monitoring-url", "u", cfg.MonitoringURL, "Readiness monitoring url")
+}

halo/cmd/ready.go

@@ -0,0 +1,66 @@
+package cmd
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/omni-network/omni/lib/errors"
+	"github.com/omni-network/omni/lib/log"
+
+	"github.com/spf13/cobra"
+)
+
+type readyConfig struct {
+	MonitoringURL string
+}
+
+func defaultReadyConfig() readyConfig {
+	return readyConfig{
+		MonitoringURL: "http://localhost:26660",
+	}
+}
+
+func newReadyCmd() *cobra.Command {
+	cfg := defaultReadyConfig()
+
+	cmd := &cobra.Command{
+		Use:   "ready",
+		Short: "Query remote node for readiness",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			err := queryReady(cmd.Context(), cfg)
+			if err != nil {
+				return errors.Wrap(err, "ready")
+			}
+
+			return nil
+		},
+	}
+
+	bindReadyFlags(cmd, &cfg)
+
+	return cmd
+}
+
+// queryReady calls halo's /ready endpoint and returns nil if the status is ready
+// or an error otherwise.
+func queryReady(ctx context.Context, cfg readyConfig) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, cfg.MonitoringURL, nil)
+	if err != nil {
+		return errors.Wrap(err, "http request creation")
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return errors.Wrap(err, "http request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode/100 != 2 {
+		return errors.New("node not ready")
+	}
+
+	log.Info(ctx, "Node ready")
+
+	return nil
+}

halo/cmd/testdata/TestCLIReference_halo.golden

@@ -8,6 +8,7 @@ Available Commands:
   consensus-pubkey Print the consensus public key
   help             Help about any command
   init             Initializes required halo files and directories
+  ready            Query remote node for readiness
   rollback         Rollback Cosmos SDK and CometBFT state by one height
   run              Runs the halo consensus client
   status           Query remote node for status

scripts/halovisor/Dockerfile

@@ -35,6 +35,8 @@ COPY --from=build-cosmovisor /ko-app/cosmovisor /usr/local/bin/cosmovisor
 COPY --from=build-0-genesis /app /halovisor/genesis/bin/halo
 COPY --from=build-1-uluwatu /app /halovisor/upgrades/1_uluwatu/bin/halo
 
+HEALTHCHECK CMD ["/halovisor/upgrades/1_uluwatu/bin/halo", "ready"]
+
 # Cosmovisor is the entrypoint
 ENTRYPOINT [ "cosmovisor" ]
 # First 'run' is cosmovisor command, second 'run' is halo command.