Merge pull request #329 from input-output-hk/refine-exe-deps

Use cardano-node as a dependency and introduce CardanoNodeSpec
cardano-scaling · May 2, 2022 · deb10cf · deb10cf
2 parents 133202a + 47f86e2
commit deb10cf
Show file tree

Hide file tree

Showing 17 changed files with 258 additions and 291 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -58,8 +58,8 @@ jobs:
 
     - name: Build
       run: |
-        nix-shell --arg withoutDevTools true --run 'cabal update'
-        nix-shell --arg withoutDevTools true --run 'cabal build ${{ matrix.package }}'
+        nix-shell --pure --arg withoutDevTools true --run 'cabal update'
+        nix-shell --pure --arg withoutDevTools true --run 'cabal build ${{ matrix.package }}'
 
     - name: Test
       if: ${{ matrix.package != 'hydra-tui' }}
@@ -76,15 +76,15 @@ jobs:
         TERM: "xterm"
       continue-on-error: true
       run: |
-        nix-shell --arg withoutDevTools true --run 'cabal test ${{ matrix.package }}'
+        nix-shell --pure --arg withoutDevTools true --run 'cabal test ${{ matrix.package }}'
 
     - name: Test (TUI, retry)
       if: steps.test_tui.outcome=='failure'
       shell: 'script -q -e -c "bash {0}"'
       env:
         TERM: "xterm"
       run: |
-        nix-shell --arg withoutDevTools true --run 'cabal test ${{ matrix.package }}'
+        nix-shell --pure --arg withoutDevTools true --run 'cabal test ${{ matrix.package }}'
 
     - name: Upload build & test artifacts
       uses: actions/upload-artifact@v2
@@ -150,16 +150,16 @@ jobs:
 
     - name: Prepare nix-shell
       run: |
-        nix-build --arg withoutDevTools true shell.nix
-        nix-shell --arg withoutDevTools true --run 'cabal update'
+        nix-build --pure --arg withoutDevTools true shell.nix
+        nix-shell --pure --arg withoutDevTools true --run 'cabal update'
 
     - name: Bench
       run: |
-        nix-shell --arg withoutDevTools true --run 'cabal bench ${{ matrix.bench }} --benchmark-options "${{ matrix.options }}"'
+        nix-shell --pure --arg withoutDevTools true --run 'cabal bench ${{ matrix.bench }} --benchmark-options "${{ matrix.options }}"'
 
     - name: Documentation (Haddock)
       run: |
-        nix-shell --arg withoutDevTools true --run '.github/workflows/ci-haddock.sh'
+        nix-shell --pure --arg withoutDevTools true --run '.github/workflows/ci-haddock.sh'
 
     - name: Upload build & test artifacts
       uses: actions/upload-artifact@v2

diff --git a/cabal.project b/cabal.project
@@ -38,7 +38,7 @@ package plutus-merkle-tree
 -- Always show detailed output for tests
 test-show-details: direct
 
--- NOTE(SN): The following is taken from cardano-node v1.31.0
+-- NOTE(SN): The following is taken from cardano-node v1.32.0-rc2
 
 source-repository-package
   type: git
@@ -48,7 +48,6 @@ source-repository-package
   subdir:
     cardano-api
     cardano-node
-    cardano-cli
 
 -- NOTE(SN): These source-repository-package tags are copied from the
 -- 'cardano-node' repository cabal.project at the revision given above. Make
@@ -192,12 +191,6 @@ source-repository-package
     stubs/plutus-ghc-stub
     word-array
 
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/ekg-forward
-  tag: 2adc8b698443bb10154304b24f6c1d6913bb65b9
-  --sha256: 0cyixq3jmq43zs1yzrycqw1klyjy0zxf1vifknnr1k9d6sc3zf6b
-
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/cardano-config

diff --git a/docs/docs/getting-started/installation.md b/docs/docs/getting-started/installation.md
@@ -63,7 +63,7 @@ You can also use `nix-build` to build the project and all executables. You will
     Do not confuse `lzma` with `liblzma-dev`, those are 2 existing package!
 
 1. Install forked libsodium
-    
+
     ````mdx-code-block
     <TerminalWindow>
 
@@ -79,6 +79,8 @@ You can also use `nix-build` to build the project and all executables. You will
     </TerminalWindow>
     ````
 
+1. To run integration tests & benchmarks, do install a recent `cardano-node` to the path. For example, see the official doc [here](https://developers.cardano.org/docs/get-started/installing-cardano-node).
+
 1. Build and test everything:
 
     ```mdx-code-block

diff --git a/hydra-cluster/bench/Bench/EndToEnd.hs b/hydra-cluster/bench/Bench/EndToEnd.hs
@@ -8,8 +8,8 @@ import Hydra.Prelude
 import Test.Hydra.Prelude
 
 import CardanoClient (submit, waitForTransaction)
-import CardanoCluster (Marked (Fuel), defaultNetworkId, newNodeConfig, seedFromFaucet, withBFTNode)
-import CardanoNode (RunningNode (..))
+import CardanoCluster (Marked (Fuel), defaultNetworkId, seedFromFaucet)
+import CardanoNode (RunningNode (..), newNodeConfig, withBFTNode)
 import Control.Lens (to, (^?))
 import Control.Monad.Class.MonadAsync (mapConcurrently)
 import Control.Monad.Class.MonadSTM (
@@ -37,7 +37,7 @@ import Hydra.Ledger (txId)
 import Hydra.Logging (withTracerOutputTo)
 import Hydra.Party (deriveParty)
 import HydraNode (
-  EndToEndLog (..),
+  EndToEndLog (FromCardanoNode),
   HydraClient,
   hydraNodeId,
   input,
@@ -81,7 +81,7 @@ bench timeoutSeconds workDir dataset@Dataset{clientDatasets} clusterSize =
           let parties = Set.fromList (deriveParty <$> hydraKeys)
           config <- newNodeConfig workDir
           withOSStats workDir $
-            withBFTNode (contramap FromCluster tracer) config $ \node@(RunningNode _ nodeSocket) -> do
+            withBFTNode (contramap FromCardanoNode tracer) config $ \node@(RunningNode _ nodeSocket) -> do
               withHydraCluster tracer workDir nodeSocket 0 cardanoKeys hydraKeys $ \(leader :| followers) -> do
                 let clients = leader : followers
                 waitForNodesConnected tracer clients

diff --git a/hydra-cluster/hydra-cluster.cabal b/hydra-cluster/hydra-cluster.cabal
@@ -116,6 +116,7 @@ library
     CardanoClient
     CardanoCluster
     CardanoNode
+    Hydra.Cluster.Util
     Hydra.Generator
     Hydra.LogFilter
     HydraNode
@@ -189,6 +190,7 @@ test-suite integration
     Paths_hydra_cluster
     Spec
     Test.CardanoClusterSpec
+    Test.CardanoNodeSpec
     Test.DirectChainSpec
     Test.EndToEndSpec
     Test.GeneratorSpec
@@ -206,6 +208,7 @@ test-suite integration
     , cardano-ledger-shelley
     , cardano-ledger-shelley-ma
     , containers
+    , directory
     , filepath
     , hedgehog-quickcheck
     , hspec
@@ -230,8 +233,7 @@ test-suite integration
     , text
 
   build-tool-depends:
-    hspec-discover:hspec-discover -any, hydra-node:hydra-node -any,
-    cardano-node:cardano-node -any, cardano-cli:cardano-cli -any
+    hspec-discover:hspec-discover -any, hydra-node:hydra-node -any
 
   ghc-options:        -threaded -rtsopts
 
@@ -266,7 +268,5 @@ benchmark bench-e2e
     , strict-containers
     , time
 
-  build-tool-depends:
-    hydra-node:hydra-node -any, cardano-node:cardano-node -any
-
+  build-tool-depends: hydra-node:hydra-node -any
   ghc-options:        -threaded -rtsopts
diff --git a/hydra-cluster/src/CardanoCluster.hs b/hydra-cluster/src/CardanoCluster.hs
@@ -18,44 +18,26 @@ import CardanoClient (
   waitForPayment,
  )
 import CardanoNode (
-  CardanoNodeArgs (
-    nodeAlonzoGenesisFile,
-    nodeByronGenesisFile,
-    nodeConfigFile,
-    nodeDlgCertFile,
-    nodeKesKeyFile,
-    nodeOpCertFile,
-    nodePort,
-    nodeShelleyGenesisFile,
-    nodeSignKeyFile,
-    nodeVrfKeyFile
-  ),
   CardanoNodeConfig (..),
   NodeId,
   NodeLog,
   Port,
   PortsConfig (..),
   RunningNode (..),
-  defaultCardanoNodeArgs,
-  withCardanoNode,
+  initSystemStart,
+  withBFTNode,
  )
-import Control.Tracer (Tracer, traceWith)
+import Control.Tracer (Tracer)
 import qualified Data.Aeson as Aeson
-import qualified Data.ByteString as BS
 import Hydra.Chain.Direct.Util (markerDatumHash, retry)
 import qualified Hydra.Chain.Direct.Util as Cardano
+import Hydra.Cluster.Util (readConfigFile)
 import Hydra.Options (
   ChainConfig (..),
   defaultChainConfig,
  )
-import qualified Paths_hydra_cluster as Pkg
-import System.Directory (createDirectoryIfMissing, doesFileExist)
 import System.FilePath ((<.>), (</>))
-import System.Posix.Files (
-  ownerReadMode,
-  setFileMode,
- )
-import Test.Network.Ports (randomUnusedTCPPort, randomUnusedTCPPorts)
+import Test.Network.Ports (randomUnusedTCPPorts)
 
 -- | TODO: This is hard-coded and must match what's in the genesis file, so
 -- ideally, we want to either:
@@ -114,7 +96,6 @@ chainConfigFor me targetDir nodeSocket them = do
   vkTarget x = targetDir </> vkName x
   skName x = actorName x <.> ".sk"
   vkName x = actorName x <.> ".vk"
-
 -- * Starting a cluster or single nodes
 
 data RunningCluster = RunningCluster ClusterConfig [RunningNode]
@@ -136,88 +117,25 @@ withCluster tr cfg@ClusterConfig{parentStateDirectory} action = do
     makeNodesConfig parentStateDirectory systemStart
       <$> randomUnusedTCPPorts 3
 
-  withBFTNode tr cfgA $ \nodeA -> do
-    withBFTNode tr cfgB $ \nodeB -> do
-      withBFTNode tr cfgC $ \nodeC -> do
+  withBFTNode (nodeTracer cfgA) cfgA $ \nodeA -> do
+    withBFTNode (nodeTracer cfgB) cfgB $ \nodeB -> do
+      withBFTNode (nodeTracer cfgC) cfgC $ \nodeC -> do
         let nodes = [nodeA, nodeB, nodeC]
         action (RunningCluster cfg nodes)
-
--- | Start a cardano-node in BFT mode using the config from config/ and
--- credentials from config/credentials/ using given 'nodeId'. NOTE: This means
--- that nodeId should only be 1,2 or 3 and that only the faucet receives
--- 'initialFunds'. Use 'seedFromFaucet' to distribute funds other wallets.
-withBFTNode ::
-  Tracer IO ClusterLog ->
-  CardanoNodeConfig ->
-  (RunningNode -> IO ()) ->
-  IO ()
-withBFTNode clusterTracer cfg action = do
-  createDirectoryIfMissing False (stateDirectory cfg)
-
-  [dlgCert, signKey, vrfKey, kesKey, opCert] <-
-    forM
-      [ dlgCertFilename nid
-      , signKeyFilename nid
-      , vrfKeyFilename nid
-      , kesKeyFilename nid
-      , opCertFilename nid
-      ]
-      (copyCredential (stateDirectory cfg))
-
-  let args =
-        defaultCardanoNodeArgs
-          { nodeDlgCertFile = Just dlgCert
-          , nodeSignKeyFile = Just signKey
-          , nodeVrfKeyFile = Just vrfKey
-          , nodeKesKeyFile = Just kesKey
-          , nodeOpCertFile = Just opCert
-          , nodePort = Just (ours (ports cfg))
-          }
-
-  readConfigFile "cardano-node.json"
-    >>= writeFileBS
-      (stateDirectory cfg </> nodeConfigFile args)
-
-  readConfigFile "genesis-byron.json"
-    >>= writeFileBS
-      (stateDirectory cfg </> nodeByronGenesisFile args)
-
-  readConfigFile "genesis-shelley.json"
-    >>= writeFileBS
-      (stateDirectory cfg </> nodeShelleyGenesisFile args)
-
-  readConfigFile "genesis-alonzo.json"
-    >>= writeFileBS
-      (stateDirectory cfg </> nodeAlonzoGenesisFile args)
-
-  withCardanoNode nodeTracer cfg args $ \rn -> do
-    traceWith clusterTracer $ MsgNodeStarting cfg
-    waitForSocket rn
-    action rn
  where
-  dlgCertFilename i = "delegation-cert.00" <> show (i - 1) <> ".json"
-  signKeyFilename i = "delegate-keys.00" <> show (i - 1) <> ".key"
-  vrfKeyFilename i = "delegate" <> show i <> ".vrf.skey"
-  kesKeyFilename i = "delegate" <> show i <> ".kes.skey"
-  opCertFilename i = "opcert" <> show i <> ".cert"
-
-  copyCredential parentDir file = do
-    bs <- readConfigFile ("credentials" </> file)
-    let destination = parentDir </> file
-    unlessM (doesFileExist destination) $
-      writeFileBS destination bs
-    setFileMode destination ownerReadMode
-    pure destination
-
-  nid = nodeId cfg
-
-  nodeTracer = contramap (MsgFromNode nid) clusterTracer
-
-  waitForSocket :: RunningNode -> IO ()
-  waitForSocket node@(RunningNode _ socket) = do
-    unlessM (doesFileExist socket) $ do
-      threadDelay 0.1
-      waitForSocket node
+  nodeTracer CardanoNodeConfig{nodeId} = contramap (MsgFromNode nodeId) tr
+
+  makeNodesConfig ::
+    FilePath ->
+    UTCTime ->
+    [Port] ->
+    (CardanoNodeConfig, CardanoNodeConfig, CardanoNodeConfig)
+  makeNodesConfig stateDirectory systemStart [a, b, c] =
+    ( CardanoNodeConfig 1 (stateDirectory </> "node-1") systemStart (PortsConfig a [b, c])
+    , CardanoNodeConfig 2 (stateDirectory </> "node-2") systemStart (PortsConfig b [a, c])
+    , CardanoNodeConfig 3 (stateDirectory </> "node-3") systemStart (PortsConfig c [a, b])
+    )
+  makeNodesConfig _ _ _ = error "we only support topology for 3 nodes"
 
 data Marked = Fuel | Normal
 
@@ -286,44 +204,6 @@ seedFromFaucet_ ::
 seedFromFaucet_ nid node vk ll marked =
   void $ seedFromFaucet nid node vk ll marked
 
--- | Initialize the system start time to now (modulo a small offset needed to
--- give time to the system to bootstrap correctly).
-initSystemStart :: IO UTCTime
-initSystemStart = do
-  addUTCTime 1 <$> getCurrentTime
-
-makeNodesConfig ::
-  FilePath ->
-  UTCTime ->
-  [Port] ->
-  (CardanoNodeConfig, CardanoNodeConfig, CardanoNodeConfig)
-makeNodesConfig stateDirectory systemStart [a, b, c] =
-  ( CardanoNodeConfig 1 (stateDirectory </> "node-1") systemStart (PortsConfig a [b, c])
-  , CardanoNodeConfig 2 (stateDirectory </> "node-2") systemStart (PortsConfig b [a, c])
-  , CardanoNodeConfig 3 (stateDirectory </> "node-3") systemStart (PortsConfig c [a, b])
-  )
-makeNodesConfig _ _ _ = error "we only support topology for 3 nodes"
-
-newNodeConfig ::
-  FilePath ->
-  IO CardanoNodeConfig
-newNodeConfig stateDirectory = do
-  nodePort <- randomUnusedTCPPort
-  systemStart <- initSystemStart
-  pure $
-    CardanoNodeConfig
-      { nodeId = 1
-      , stateDirectory
-      , systemStart
-      , ports = PortsConfig nodePort []
-      }
-
--- | Lookup a config file similar reading a file from disk.
-readConfigFile :: FilePath -> IO ByteString
-readConfigFile source = do
-  filename <- Pkg.getDataFileName ("config" </> source)
-  BS.readFile filename
-
 --
 -- Logging
 --