Merge #4105

4105: RTView: MVP r=denisshevchenko a=denisshevchenko

Closes #4096 

MVP for RTView: it's not ready for production yet, but ready for testing by SPOs.

Co-authored-by: Denis Shevchenko <[email protected]>

cardano-node/src/Cardano/Node/Run.hs

@@ -3,6 +3,7 @@
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE FlexibleContexts #-}
 {-# LANGUAGE GADTs #-}
+{-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE PackageImports #-}
 {-# LANGUAGE ScopedTypeVariables #-}
@@ -200,6 +201,11 @@ handleNodeWithTracers cmdPc nc p networkMagic runP = do
               networkMagic
               nodeKernelData
               p2pMode
+
+          startupInfo <- getStartupInfo nc p fp
+          mapM_ (traceWith $ startupTracer tracers) startupInfo
+          traceNodeStartupInfo (nodeStartupInfoTracer tracers) startupInfo
+
           handleSimpleNode runP p2pMode tracers nc
             (\nk -> do
                 setNodeKernel nodeKernelData nk
@@ -249,6 +255,17 @@ handleNodeWithTracers cmdPc nc p networkMagic runP = do
                     forM_ eLoggingLayer
                       shutdownLoggingLayer
 
+-- | Currently, we trace only 'ShelleyBased'-info which will be asked
+--   by 'cardano-tracer' service as a datapoint. It can be extended in the future.
+traceNodeStartupInfo
+  :: Tracer IO NodeStartupInfo
+  -> [StartupTrace blk]
+  -> IO ()
+traceNodeStartupInfo t startupTrace =
+  forM_ startupTrace $ \case
+    BIShelley (BasicInfoShelleyBased era _ sl el spkp) ->
+      traceWith t $ NodeStartupInfo era sl el spkp
+    _ -> return ()
 
 logTracingVerbosity :: NodeConfiguration -> Tracer IO String -> IO ()
 logTracingVerbosity nc tracer =

cardano-node/src/Cardano/Node/Startup.hs

@@ -209,3 +209,25 @@ prepareNodeInfo ptcl (SomeConsensusProtocol whichP pForInfo) tc nodeStartTime =
     case tcNodeName tc of
       Just aName -> return aName
       Nothing    -> pack <$> getHostName
+
+-- | This information is taken from 'BasicInfoShelleyBased'. It is required for
+--   'cardano-tracer' service (particularly, for RTView).
+data NodeStartupInfo = NodeStartupInfo {
+    suiEra               :: Text
+  , suiSlotLength        :: NominalDiffTime
+  , suiEpochLength       :: Word64
+  , suiSlotsPerKESPeriod :: Word64
+  } deriving (Eq, Generic, ToJSON, FromJSON, Show)
+
+docNodeStartupInfoTraceEvent :: Documented NodeStartupInfo
+docNodeStartupInfoTraceEvent = Documented
+  [ DocMsg
+      ["NodeStartupInfo"]
+        []
+        "Startup information about this node, required for RTView\
+        \\n\
+        \\n _suiEra_: Name of the current era. \
+        \\n _suiSlotLength_: Slot length, in seconds. \
+        \\n _suiEpochLength_: Epoch length, in slots. \
+        \\n _suiSlotsPerKESPeriod_: KES period length, in slots."
+  ]

cardano-node/src/Cardano/Node/Tracing.hs

@@ -21,7 +21,7 @@ import Ouroboros.Network.NodeToNode   (NodeToNodeVersion, RemoteAddress)
 import Ouroboros.Network.NodeToClient (LocalAddress, NodeToClientVersion)
 
 import Cardano.Node.Handlers.Shutdown (ShutdownTrace)
-import Cardano.Node.Startup           (NodeInfo, StartupTrace)
+import Cardano.Node.Startup           (NodeInfo, NodeStartupInfo, StartupTrace)
 
 import Cardano.Logging.Resources
 import Cardano.Node.Tracing.StateRep (NodeState)
@@ -46,6 +46,7 @@ data Tracers peer localPeer blk p2p = Tracers
   , startupTracer         :: Tracer IO (StartupTrace blk)
   , shutdownTracer        :: Tracer IO ShutdownTrace
   , nodeInfoTracer        :: Tracer IO NodeInfo
+  , nodeStartupInfoTracer :: Tracer IO NodeStartupInfo
   , nodeStateTracer       :: Tracer IO NodeState
   , resourcesTracer       :: Tracer IO ResourceStats
   , peersTracer           :: Tracer IO [PeerT blk]

cardano-node/src/Cardano/Node/Tracing/Documentation.hs

@@ -196,6 +196,13 @@ docTracers configFileName outputFileName _ _ _ = do
     nodeInfoTrDoc <- documentTracer trConfig nodeInfoTr
       (docNodeInfoTraceEvent :: Documented NodeInfo)
 
+    nodeStartupInfoTr <- mkDataPointTracer
+                trDataPoint
+                (const ["NodeStartupInfo"])
+    configureTracers trConfig docNodeStartupInfoTraceEvent [nodeStartupInfoTr]
+    nodeStartupInfoTrDoc <- documentTracer trConfig nodeStartupInfoTr
+      (docNodeStartupInfoTraceEvent :: Documented NodeStartupInfo)
+
     -- State tracer
     stateTr   <- mkCardanoTracer
                 trBase trForward mbTrEKG
@@ -865,6 +872,7 @@ docTracers configFileName outputFileName _ _ _ = do
 
     let bl =  nodeInfoTrDoc
             <> stateTrDoc
+            <> nodeStartupInfoTrDoc
             <> resourcesTrDoc
             <> startupTrDoc
             <> shutdownTrDoc

cardano-node/src/Cardano/Node/Tracing/Tracers.hs

@@ -94,6 +94,11 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig enabl
                 (const ["NodeInfo"])
     configureTracers trConfig docNodeInfoTraceEvent [nodeInfoDP]
 
+    nodeStartupInfoDP <- mkDataPointTracer
+                trDataPoint
+                (const ["NodeStartupInfo"])
+    configureTracers trConfig docNodeStartupInfoTraceEvent [nodeStartupInfoDP]
+
     nodeStateDP <- mkDataPointTracer
                 trDataPoint
                 (const ["NodeState"])
@@ -217,6 +222,7 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig enabl
       , shutdownTracer  = Tracer (traceWith shutdownTr)
                         <> Tracer (SR.traceNodeStateShutdown nodeStateDP)
       , nodeInfoTracer  = Tracer (traceWith nodeInfoDP)
+      , nodeStartupInfoTracer = Tracer (traceWith nodeStartupInfoDP)
       , nodeStateTracer = Tracer (traceWith stateTr)
                         <> Tracer (traceWith nodeStateDP)
       , resourcesTracer = Tracer (traceWith resourcesTr)

cardano-node/src/Cardano/Tracing/Tracers.hs

@@ -155,6 +155,7 @@ nullTracersP2P = Tracers
   , startupTracer = nullTracer
   , shutdownTracer = nullTracer
   , nodeInfoTracer = nullTracer
+  , nodeStartupInfoTracer = nullTracer
   , nodeStateTracer = nullTracer
   , resourcesTracer = nullTracer
   , peersTracer = nullTracer
@@ -171,6 +172,7 @@ nullTracersNonP2P = Tracers
   , startupTracer = nullTracer
   , shutdownTracer = nullTracer
   , nodeInfoTracer = nullTracer
+  , nodeStartupInfoTracer = nullTracer
   , nodeStateTracer = nullTracer
   , resourcesTracer = nullTracer
   , peersTracer = nullTracer
@@ -333,6 +335,7 @@ mkTracers blockConfig tOpts@(TracingOnLegacy trSel) tr nodeKern ekgDirect enable
     , shutdownTracer = toLogObject' verb $ appendName "shutdown" tr
     -- The remaining tracers are completely unused by the legacy tracing:
     , nodeInfoTracer = nullTracer
+    , nodeStartupInfoTracer = nullTracer
     , nodeStateTracer = nullTracer
     , resourcesTracer = nullTracer
     , peersTracer = nullTracer
@@ -475,6 +478,7 @@ mkTracers _ _ _ _ _ enableP2P =
     , startupTracer = nullTracer
     , shutdownTracer = nullTracer
     , nodeInfoTracer = nullTracer
+    , nodeStartupInfoTracer = nullTracer
     , nodeStateTracer = nullTracer
     , resourcesTracer = nullTracer
     , peersTracer = nullTracer

cardano-tracer/configuration/complete-example.json

@@ -1,8 +1,8 @@
 {
   "networkMagic": 764824073,
   "network": {
-    "tag": "AcceptAt",
-    "contents": "/tmp/forwarder.sock"
+    "tag": "ConnectTo",
+    "contents": ["/tmp/forwarder.sock"]
   },
   "loRequestNum": 100,
   "ekgRequestFreq": 2,
@@ -20,6 +20,10 @@
     "epHost": "127.0.0.1",
     "epPort": 3000
   },
+  "hasRTView": {
+    "epHost": "127.0.0.1",
+    "epPort": 3300
+  },
   "logging": [
     {
       "logRoot": "/tmp/cardano-tracer-h-logs",

cardano-tracer/configuration/complete-example.yaml

@@ -1,8 +1,9 @@
 ---
 networkMagic: 764824073
 network:
-  tag: AcceptAt
-  contents: "/tmp/forwarder.sock"
+  tag: ConnectTo
+  contents:
+  - "/tmp/forwarder.sock"
 loRequestNum: 100
 ekgRequestFreq: 2
 hasEKG:
@@ -13,6 +14,9 @@ hasEKG:
 hasPrometheus:
   epHost: 127.0.0.1
   epPort: 3000
+hasRTView:
+  epHost: 127.0.0.1
+  epPort: 3300
 logging:
 - logRoot: "/tmp/cardano-tracer-h-logs"
   logMode: FileMode

cardano-tracer/configuration/minimal-example.json

@@ -1,8 +1,8 @@
 {
   "networkMagic": 764824073,
   "network": {
-    "tag": "ConnectTo",
-    "contents": ["/tmp/forwarder.sock"]
+    "tag": "AcceptAt",
+    "contents": "/tmp/forwarder.sock"
   },
   "logging": [
     {

cardano-tracer/configuration/minimal-example.yaml

@@ -1,9 +1,8 @@
 ---
 networkMagic: 764824073
 network:
-  tag: ConnectTo
-  contents:
-  - "/tmp/forwarder.sock"
+  tag: AcceptAt
+  contents: "/tmp/forwarder.sock"
 logging:
 - logRoot: "/tmp/cardano-tracer-logs"
   logMode: FileMode

cardano-tracer/docs/cardano-rtview.md

@@ -0,0 +1,114 @@
+# Cardano RTView
+
+RTView is a part of `cardano-tracer` [service](https://github.com/input-output-hk/cardano-node/blob/master/cardano-tracer/docs/cardano-tracer.md). It is a real-time monitoring tool for Cardano nodes (RTView is an abbreviation for "Real Time View"). It provides an interactive web page where you can see different kinds of information about connected nodes.
+
+# Contents
+
+1. [Introduction](#Introduction)
+   1. [Motivation](#Motivation)
+   2. [Overview](#Overview)
+2. [Configuration](#Configuration)
+3. [Notifications](#Notifications)
+   1. [SMTP settings](#SMTP-settings)
+   2. [Note for Gmail users](#Note-for-Gmail-users)
+   3. [Events](#Events)
+   4. [Notify period](#Notify-period)
+4. [UI](#UI)
+   1. [Security Alert](#Security-Alert)
+
+# Introduction
+
+## Motivation
+
+For a long time, Stake Pool Operators used third-party tools for monitoring their Cardano nodes, such as [Grafana](https://grafana.com/grafana/dashboards/12469)-based installations. These third-party solutions work, but they have two main problems:
+
+1. Complex setup, especially for non-technical person.
+2. Limited kinds of displayed information. For example, metrics can be shown, but error messages cannot.
+
+RTView solves both of them:
+
+1. Its setup is as simple as possible: if you have `cardano-tracer` installed, you already have RTView.
+2. Because of using special network protocols integrated into the node, RTView can display any information that the node can provide.
+
+## Overview
+
+You can think of RTView as a SPA (Single Page Application) that can be opened in any modern browser. All the information on it changes dynamically, so you shouldn't refresh it.
+
+When you open it for the first time, you'll see a help message about required configuration.
+
+After your node connects to `cardano-tracer`, you'll see a column with different information, such as the node's name, version, protocol, era, sync percentage, KES values, blockchain info, etc. There is a separate column for each connected node, so you can see and compare their data.
+
+Also, there are dynamic charts for different metrics, such as system metrics, blockchain metrics, transaction metrics, etc.
+
+# Configuration
+
+Since RTView is a part of `cardano-tracer`, the only thing you need to do is to enable RTView (because it's disabled by default). To do it, please add the following lines to `cardano-tracer`'s configuration file.
+
+If you use `json`-configuration:
+
+```
+"hasRTView": {
+  "epHost": "127.0.0.1",
+  "epPort": 3300
+}
+```
+
+Or, if you use `yaml`-configuration:
+
+```
+hasRTView:
+  epHost: 127.0.0.1
+  epPort: 3300
+```
+
+Here `epHost` and `epPort` specify the host and the port for RTView web page.
+
+That's it. Now run `cardano-tracer` and open [127.0.0.1:3300](https://127.0.0.1:3300) in your browser.
+
+# Notifications
+
+RTView can send notifications about specified events (for example, warnings or errors). Click on the bell icon on the top bar to see the corresponding settings.
+
+## SMTP settings
+
+Technically, RTView contains an email client that sends emails using SMTP. That's why you need the SMTP settings of your email provider. Please fill in all the inputs marked with an asterisk in `Settings` window.
+
+You can use `Send test email` button to check if your email settings are correct.
+
+## Note for Gmail users
+
+If you want to set up email notifications using your Gmail account, please make sure that `2-Step Verification` is enabled. You can check it in `Google Account` -> `Security`. After you enabled `2-Step Verification`, please generate the new app password (if you don't have one already) in `Security` -> `App passwords`. You'll need this app password for RTView settings.
+
+Now you can set up RTView notifications:
+
+1. `SMTP host`: `smtp.gmail.com`
+
+2. `SMTP port`: `587`
+
+3. `Username`: most likely, it's your email address
+
+4. `Password`: app password you've generated
+
+5. `SSL`: `STARTTLS`
+
+## Events
+
+When you click on the bell icon on the top bar, you can open `Events` window. Here you can specify events you want to be notified about.
+
+For example, let's have a look at `Warnings` (i.e. all the messages from the node with `Warning` severity level). By default, the corresponding switch is disabled, which means that you won't be notified about warnings at all. But if you enable that switch, you will periodically receive a notification about warnings, if any.
+
+You can use a switch `All events` in the bottom of the window to enable/disable all the events at once. Please note that if you disable all the events, the bell icon on the top bar becomes "slashed".
+
+## Notify period
+
+You can specify how frequently you want to receive notifications for a specific event. To do it, select a value from the dropdown list at the right of the event switch. There are values from `Immediately` to `Every 12 hours`.
+
+If you selected `Immediately`, the new email with the associated event(s) will be sent right away. It can be used for critical events: most likely, you want to know about such events as soon as possible.
+
+If you selected `Every 12 hours`, the new email with associated event(s) will be sent only two times a day. I can be used for non-critical events, like `Warnings`.
+
+# UI
+
+## Security Alert
+
+When you open the web page for the first time, you'll see a warning from your browser, something like "Potential Security Risk Ahead" or "Your connection is not private". This is because `https`-connection between your browser and RTView is using [self-signed certificate](https://en.wikipedia.org/wiki/Self-signed_certificate) generated by [openssl](https://www.openssl.org/) program. So click to `Advanced` button and open the page. Technically, there is no risk at all - your connection **is** private.

cardano-tracer/docs/cardano-tracer.md

@@ -18,6 +18,7 @@
    7. [Prometheus](#Prometheus)
    8. [EKG Monitoring](#EKG-monitoring)
    9. [Verbosity](#Verbosity)
+   10. [RTView](#RTView)
 
 # Introduction
 
@@ -401,3 +402,7 @@ The optional field `verbosity` specifies the verbosity level for the `cardano-tr
 3. `Maximum` - all the messages will be shown in standard output. **Caution**: the number of messages can be huge.
 
 Please note that if you skip this field, `ErrorsOnly` verbosity will be used by default.
+
+## RTView
+
+It is a real-time monitoring tool for Cardano nodes (RTView is an abbreviation for "Real Time View"). It provides an interactive web page where you can see different kinds of information about connected nodes. Please read its documentation [here](https://github.com/input-output-hk/cardano-node/blob/master/cardano-tracer/docs/cardano-rtview.md).

cardano-tracer/src/Cardano/Tracer/Handlers/RTView/Notifications/Send.hs

@@ -5,6 +5,7 @@ module Cardano.Tracer.Handlers.RTView.Notifications.Send
   ( makeAndSendNotification
   ) where
 
+import           Control.Concurrent.Extra (Lock)
 import           Control.Concurrent.STM (atomically)
 import           Control.Concurrent.STM.TBQueue (flushTBQueue)
 import           Control.Concurrent.STM.TVar (TVar, modifyTVar', readTVarIO)
@@ -27,18 +28,19 @@ import           Cardano.Tracer.Utils
 
 makeAndSendNotification
   :: DataPointRequestors
+  -> Lock
   -> TVar UTCTime
   -> EventsQueue
   -> IO ()
-makeAndSendNotification dpRequestors lastTime eventsQueue = do
+makeAndSendNotification dpRequestors currentDPLock lastTime eventsQueue = do
   emailSettings <- readSavedEmailSettings
   unless (incompleteEmailSettings emailSettings) $ do
     events <- atomically $ nub <$> flushTBQueue eventsQueue
     let (nodeIds, tss) = unzip $ nub [(nodeId, ts) | Event nodeId ts _ _ <- events]
     unless (null nodeIds) $ do
       nodeNames <-
         forM nodeIds $ \nodeId@(NodeId anId) ->
-          askDataPoint dpRequestors nodeId "NodeInfo" >>= \case
+          askDataPoint dpRequestors currentDPLock nodeId "NodeInfo" >>= \case
             Nothing -> return anId
             Just ni -> return $ niName ni
       lastEventTime <- readTVarIO lastTime