Add retry for packet relay failures and relayer restructure (#547)

* Add retry, collect events and msgs in Link

* Flatten Connection and Channel structs and use them from CLIs

* some renaming

* Remove MsgCollector and use the Link from the CLIs.

Cleanup the CLI and make them consistent with the handshake CLIs.

* relayer/client bit of cleanup

* Restructure relayer code

* Add retries in packet relayer loop

* Cleanup packet CLIs and relayer loop output

* Fix for abiding by log_level in start command

* Add cli_utils for chain validation, change params to ChainId, add required

* Change start CLI to take the two chain-ids

* Refactored error hierarchy in relayer objects

* Remove panics from event processing, update instructions

* Add relayer listen command in doc

Co-authored-by: Adi Seredinschi <[email protected]>

CHANGELOG.md

@@ -10,6 +10,8 @@
 - [relayer-cli]
   - Implement command to query the channels associated with a connection ([#505])
   - JSON output for queries and txs ([#500])
+- [relayer]
+  - Added retry mechanism, restructured relayer ([#519])
 
 ### IMPROVEMENTS
 
@@ -39,6 +41,7 @@
 - [modules]
   - The `ibc::handler::Event` is removed and handlers now produce `ibc::events::IBCEvent`s ([#535])
 
+[#32]: https://github.com/informalsystems/ibc-rs/issues/32
 [#94]: https://github.com/informalsystems/ibc-rs/issues/94
 [#306]: https://github.com/informalsystems/ibc-rs/issues/306
 [#470]: https://github.com/informalsystems/ibc-rs/issues/470
@@ -49,13 +52,13 @@
 [#513]: https://github.com/informalsystems/ibc-rs/issues/513
 [#514]: https://github.com/informalsystems/ibc-rs/issues/514
 [#517]: https://github.com/informalsystems/ibc-rs/issues/517
+[#519]: https://github.com/informalsystems/ibc-rs/issues/519
 [#525]: https://github.com/informalsystems/ibc-rs/issues/525
 [#527]: https://github.com/informalsystems/ibc-rs/issues/527
-[#32]: https://github.com/informalsystems/ibc-rs/issues/32
+[#535]: https://github.com/informalsystems/ibc-rs/issues/535
 [#536]: https://github.com/informalsystems/ibc-rs/issues/536
 [#537]: https://github.com/informalsystems/ibc-rs/issues/537
 [#540]: https://github.com/informalsystems/ibc-rs/issues/540
-[#535]: https://github.com/informalsystems/ibc-rs/issues/535
 
 
 ## v0.0.6

dev-env

@@ -39,6 +39,16 @@ if ! [ -f $CONFIG_FILE ]; then
   usage
 fi
 
+if ! grep -q -s "$CHAIN_0_ID" "$CONFIG_FILE"; then
+  echo "error: configuration for chain [$CHAIN_0_ID] does not exist in file $CONFIG_FILE."
+  usage
+fi
+
+if ! grep -q -s "$CHAIN_1_ID" "$CONFIG_FILE"; then
+  echo "error: configuration for chain [$CHAIN_1_ID] does not exist in file $CONFIG_FILE."
+  usage
+fi
+
 GAIA_DATA="$(pwd)/data"
 
 # Ensure user understands what will be deleted

modules/src/ics02_client/events.rs

@@ -46,7 +46,7 @@ fn extract_attributes_from_tx(event: &tendermint::abci::Event) -> Attributes {
             CLIENT_TYPE_ATTRIBUTE_KEY => attr.client_type = value.parse().unwrap(),
             CONSENSUS_HEIGHT_ATTRIBUTE_KEY => attr.consensus_height = value.parse().unwrap(),
             // TODO: `Attributes` has 4 fields and we're only parsing 3
-            _ => panic!("unexpected attribute key: {}", key),
+            _ => {}
         }
     }
 

modules/src/ics03_connection/events.rs

@@ -53,7 +53,7 @@ fn extract_attributes_from_tx(event: &tendermint::abci::Event) -> Attributes {
                 attr.counterparty_client_id = value.parse().unwrap()
             }
             // TODO: `Attributes` has 5 fields and we're only parsing 4
-            _ => panic!("unexpected attribute key: {}", key),
+            _ => {}
         }
     }
 

modules/src/ics04_channel/events.rs

@@ -120,8 +120,7 @@ fn extract_attributes_from_tx(event: &tendermint::abci::Event) -> Attributes {
             COUNTERPARTY_CHANNEL_ID_ATTRIBUTE_KEY => {
                 attr.counterparty_channel_id = value.parse().ok()
             }
-            // TODO: `Attributes` has 6 fields and we're only parsing 5
-            _ => panic!("unexpected attribute key: {}", key),
+            _ => {}
         }
     }
 
@@ -146,7 +145,7 @@ fn extract_packet_and_write_ack_from_tx(
             PKT_DATA_ATTRIBUTE_KEY => packet.data = Vec::from(value.as_bytes()),
             // TODO: `Packet` has 7 fields and we're only parsing 6; is that intended?
             PKT_ACK_ATTRIBUTE_KEY => write_ack = Some(Vec::from(value.as_bytes())),
-            _ => panic!("unexpected attribute key: {}", key),
+            _ => {}
         };
     }
 

one-chain

@@ -125,7 +125,7 @@ else
 fi
 
 # Start gaia
-$BINARY --home $CHAIN_DIR/$CHAIN_ID start --trace --pruning=nothing --grpc.address="0.0.0.0:$GRPC_PORT" > $CHAIN_DIR/$CHAIN_ID.log 2>&1 &
+$BINARY --home $CHAIN_DIR/$CHAIN_ID start --pruning=nothing --grpc.address="0.0.0.0:$GRPC_PORT" > $CHAIN_DIR/$CHAIN_ID.log 2>&1 &
 
 # Start gaia with trace
 # $BINARY --home $CHAIN_DIR/$CHAIN_ID start --trace --pruning=nothing --grpc.address="0.0.0.0:$GRPC_PORT" > $CHAIN_DIR/$CHAIN_ID.log 2>&1 &

relayer-cli/relayer_operation_instructions.md

@@ -17,9 +17,18 @@
     e.g.:
 
     ```shell
-    ./dev-env my_config.toml ibc-0 ibc-1
+    ./dev-env loop_config.toml ibc-0 ibc-1
     ```
 
+#### Stop and cleanup
+
+You can manually stop the two gaia instances and cleanup after them as follows:
+
+```shell
+killall gaiad
+rm -rf data/
+```
+
 ### CLI Step Relaying:
 
 You can use the relayer CLIs, below are some examples.
@@ -28,7 +37,7 @@ You can use the relayer CLIs, below are some examples.
 can be executed as `rrly`, eg. by using an shell alias:
 
 ```shell script
-alias rrly='cargo run --bin relayer --'`
+alias rrly='cargo run --bin relayer --'
 ```
 
 #### Client CLIs:
@@ -138,7 +147,7 @@ Note that the addresses used in the two commands above are configured in `dev-en
 
 First, we'll send 9999 samoleans from `ibc-0` to `ibc-1`.
 
-- send 1 packet to ibc-0
+- start the transfer of 9999 samoleans from `ibc-0` to `ibc-1`. This results in a Tx to `ibc-0` for a `MsgTransfer` packet
 
     ```shell script
     rrly -c loop_config.toml tx raw packet-send ibc-0 ibc-1 transfer channel-0 9999 1000 -n 1 -d samoleans
@@ -153,7 +162,7 @@ First, we'll send 9999 samoleans from `ibc-0` to `ibc-1`.
 - send recv_packet to ibc-1
 
     ```shell script
-    rrly -c loop_config.toml tx raw packet-recv ibc-0 ibc-1 07-tendermint-0 07-tendermint-0 transfer transfer channel-0 channel-0
+    rrly -c loop_config.toml tx raw packet-recv ibc-1 ibc-0 transfer channel-0
     ```
 
 - query unreceived acks on ibc-0
@@ -165,7 +174,7 @@ First, we'll send 9999 samoleans from `ibc-0` to `ibc-1`.
 - send acknowledgement to ibc-0
 
     ```shell script
-    rrly -c loop_config.toml tx raw packet-ack  ibc-0 ibc-1 07-tendermint-0 07-tendermint-0 transfer transfer channel-0 channel-0
+    rrly -c loop_config.toml tx raw packet-ack  ibc-0 ibc-1 transfer channel-0
     ```
 
 - send 1 packet with low timeout height offset to ibc-0
@@ -177,38 +186,59 @@ First, we'll send 9999 samoleans from `ibc-0` to `ibc-1`.
 - send timeout to ibc-0
 
     ```shell script
-    rrly -c loop_config.toml tx raw packet-recv ibc-0 ibc-1 07-tendermint-0 07-tendermint-0 transfer transfer channel-0 channel-0
+    rrly -c loop_config.toml tx raw packet-recv ibc-1 ibc-0 transfer channel-0
     ```
 
-Now, we'll send those samoleans back, from `ibc-1` to `ibc-1`.
+Send those samoleans back, from `ibc-1` to `ibc-1`.
 
 ```shell script
 rrly -c loop_config.toml tx raw packet-send ibc-1 ibc-0 transfer channel-0 9999 1000 -n 1 -d ibc/27A6394C3F9FF9C9DCF5DFFADF9BB5FE9A37C7E92B006199894CF1824DF9AC7C
-rrly -c loop_config.toml tx raw packet-recv ibc-1 ibc-0 07-tendermint-0 07-tendermint-0 transfer transfer channel-0 channel-0
-rrly -c loop_config.toml tx raw packet-ack  ibc-1 ibc-0 07-tendermint-0 07-tendermint-0 transfer transfer channel-0 channel-0
+rrly -c loop_config.toml tx raw packet-recv ibc-0 ibc-1 transfer channel-0
+rrly -c loop_config.toml tx raw packet-ack  ibc-1 ibc-0 transfer channel-0
 ```
 
 The `ibc/27A6394C3F9FF9C9DCF5DFFADF9BB5FE9A37C7E92B006199894CF1824DF9AC7C` denominator above can be obtained by querying the balance at `ibc-1` after the transfer from `ibc-0` to `ibc-1` is concluded.
 
 ### Relayer loop:
 
-Client, connection, channel handshake and packet relaying can pe done from the relayer loop
+Client, connection, channel handshake and packet relaying can pe done from 
+the relayer `v0` loop.
 
-- start the relayer, the relayer should create the clients, and do the handshake for the connection and channel. Once that is done it will sit in a loop, listening for events
+- start the relayer
+    - with new channel:
 
-    ```shell script
-    rrly -c loop_config.toml v-0
-    ```
+        ```shell script
+        rrly -c loop_config.toml start ibc-0 ibc-1
+        ```
+      The relayer should create the clients, and perform the handshake for new clients, connection and channel between the two chains on `transfer` port. Once that is finished, it listens for IBC packet events and relays receive packets, acknowledgments and timeouts.
 
-- use the CLI to send 2 packets to ibc0 chain:
+      Note: The configuration file should have the relay path specified, for example:
+      ```
+        [[connections]]
+        a_chain = 'ibc-0'
+        b_chain = 'ibc-1'
+
+        [[connections.paths]]
+        a_port = 'transfer'
+        b_port = 'transfer'
+      ```
+
+    - with existing channel:
+
+      ```shell script
+      rrly -c loop_config.toml start ibc-0 ibc-1 transfer channel-0
+      ```
+      The relayer listens for IBC packet events over the specified channel and relays receive packets, acknowledgments and timeouts.
+
+- in a separate terminal, use the CLI to send 2 packets to ibc0 chain:
 
     ```shell script
     rrly -c loop_config.toml tx raw packet-send ibc-0 ibc-1 transfer channel-0 9999 1000 -n 2
     ```
 - use the CLI to send 2 packets to ibc0 chain:
 
     ```shell script
-    rrly -c loop_config.toml tx raw packet-send ibc-1 ibc-2 transfer channel-0 9999 1000 -n 2
+    rrly -c loop_config.toml tx raw packet-send ibc-1 ibc-0 transfer channel-0 9999 1000 -n 2
     ```
 
 - observe the output on the relayer terminal, verify that the send events are processed and the recv_packets are sent out.
@@ -222,6 +252,16 @@ Client, connection, channel handshake and packet relaying can pe done from the r
     rrly -c loop_config.toml query packet unreceived-acks ibc-1 ibc-0 transfer channel-0
     ```
 
+## Relayer listen mode
+
+The relayer can be started in listen mode:
+
+```shell script
+rrly -c loop_config.toml listen ibc-0
+```
+
+It displays the `NewBlock` and IBC events received from the specified chain.
+
 ## Profiling the relayer
 
 The `relayer` crate provides a `time!` macro which can be used to measure how much time is spent between the invocation of the macro and the end of the enclosing scope.

relayer-cli/src/application.rs

@@ -2,6 +2,7 @@
 
 use crate::components::Tracing;
 use crate::{commands::CliCmd, config::Config};
+
 use abscissa_core::terminal::component::Terminal;
 use abscissa_core::{
     application::{self, AppCell},
@@ -114,9 +115,25 @@ impl Application for CliApp {
             .map(|path| self.load_config(&path))
             .transpose()?
             .unwrap_or_default();
-        let tracing = Tracing::new(config.global)?;
 
-        Ok(vec![Box::new(terminal), Box::new(tracing)])
+        // For `start` cmd exclusively we disable JSON; otherwise output is JSON-only
+        let json_on = if let Some(c) = &command.command {
+            !matches!(c, CliCmd::Start(..))
+        } else {
+            true
+        };
+
+        if json_on {
+            let tracing = Tracing::new(config.global)?;
+            Ok(vec![Box::new(terminal), Box::new(tracing)])
+        } else {
+            let alt_tracing = abscissa_core::trace::Tracing::new(
+                abscissa_core::trace::Config::from(config.global.log_level),
+                abscissa_core::terminal::ColorChoice::Auto,
+            )
+            .unwrap();
+            Ok(vec![Box::new(terminal), Box::new(alt_tracing)])
+        }
     }
 
     /// Get tracing configuration from command-line options

relayer-cli/src/commands.rs

@@ -5,6 +5,7 @@
 //! See the `impl Configurable` below for how to specify the path to the
 //! application's configuration file.
 
+mod cli_utils;
 mod config;
 mod keys;
 mod light;

relayer-cli/src/commands/cli_utils.rs

@@ -0,0 +1,59 @@
+use abscissa_core::config;
+
+use ibc::ics24_host::identifier::ChainId;
+use relayer::chain::handle::ChainHandle;
+use relayer::chain::runtime::ChainRuntime;
+use relayer::chain::CosmosSDKChain;
+
+use crate::application::CliApp;
+use crate::error::{Error, Kind};
+
+/// Pair of chain handlers that are used by most CLIs.
+pub struct ChainHandlePair {
+    /// Source chain handle
+    pub src: Box<dyn ChainHandle>,
+    /// Destination chain handle
+    pub dst: Box<dyn ChainHandle>,
+}
+
+/// Create the source and destination chain handlers from the configuration and chain identifiers
+pub fn chain_handlers_from_chain_id(
+    config: &config::Reader<CliApp>,
+    src_chain_id: &ChainId,
+    dst_chain_id: &ChainId,
+) -> Result<ChainHandlePair, Error> {
+    let src_config = config
+        .find_chain(src_chain_id)
+        .ok_or_else(|| "missing source chain in configuration file".to_string());
+
+    let dst_config = config
+        .find_chain(dst_chain_id)
+        .ok_or_else(|| "missing destination chain configuration file".to_string());
+
+    let (src_chain_config, dst_chain_config) = match (src_config, dst_config) {
+        (Ok(s), Ok(d)) => (s, d),
+        (Err(e), _) | (_, Err(e)) => {
+            return Err(Kind::Config.context(e).into());
+        }
+    };
+
+    let src_chain_res = ChainRuntime::<CosmosSDKChain>::spawn(src_chain_config.clone())
+        .map_err(|e| Kind::Runtime.context(e));
+    let src = match src_chain_res {
+        Ok((handle, _)) => handle,
+        Err(e) => {
+            return Err(e.into());
+        }
+    };
+
+    let dst_chain_res = ChainRuntime::<CosmosSDKChain>::spawn(dst_chain_config.clone())
+        .map_err(|e| Kind::Runtime.context(e));
+    let dst = match dst_chain_res {
+        Ok((handle, _)) => handle,
+        Err(e) => {
+            return Err(e.into());
+        }
+    };
+
+    Ok(ChainHandlePair { src, dst })
+}

relayer-cli/src/commands/query/client.rs

@@ -59,12 +59,12 @@ impl QueryClientStateCmd {
     }
 }
 
-/// Command for handling a query for a client's state.
+/// Command for querying a client's state.
 /// To run with proof:
-/// cargo run --bin relayer -- -c relayer/tests/config/fixtures/simple_config.toml query client state ibc-test ethbridge --height 3
+/// rrly -c cfg.toml query client state ibc-1 07-tendermint-0 --height 3
 ///
 /// Run without proof:
-/// cargo run --bin relayer -- -c relayer/tests/config/fixtures/simple_config.toml query client state ibc-test ethbridge --height 3 -p false
+/// rrly -c cfg.toml query client state ibc-1 07-tendermint-0 --height 3 -p false
 impl Runnable for QueryClientStateCmd {
     fn run(&self) {
         let config = app_config();
@@ -153,10 +153,10 @@ impl QueryClientConsensusCmd {
 
 /// Implementation of the query for a client's consensus state at a certain height.
 /// Run with proof:
-/// cargo run --bin relayer -- -c simple_config.toml query client consensus ibc0 ibconeclient 22
+/// rrly -c cfg.toml query client consensus ibc-0 ibconeclient 22
 ///
 /// Run without proof:
-/// cargo run --bin relayer -- -c simple_config.toml query client consensus ibc0 ibconeclient 22 -p false
+/// rrly -c cfg.toml query client consensus ibc-0 ibconeclient 22 -p false
 impl Runnable for QueryClientConsensusCmd {
     fn run(&self) {
         let config = app_config();