fix: add syncing check for gRPC node (#3833)

* Add syncing check for gRPC node.

* Fix comment.

* Add changelog

.changelog/unreleased/improvements/3814-grpc-syncing.md

@@ -0,0 +1,3 @@
+- Add syncing check for gRPC node ([#3814])
+
+[#3814]: https://github.com/informalsystems/ibc-rs/issues/3814

crates/relayer/src/chain/cosmos.rs

@@ -16,6 +16,8 @@ use tonic::metadata::AsciiMetadataValue;
 use tracing::{debug, error, info, instrument, trace, warn};
 
 use ibc_proto::cosmos::base::node::v1beta1::ConfigResponse;
+use ibc_proto::cosmos::base::tendermint::v1beta1::service_client::ServiceClient;
+use ibc_proto::cosmos::base::tendermint::v1beta1::{GetSyncingRequest, GetSyncingResponse};
 use ibc_proto::cosmos::staking::v1beta1::Params as StakingParams;
 use ibc_proto::ibc::apps::fee::v1::{
     QueryIncentivizedPacketRequest, QueryIncentivizedPacketResponse,
@@ -628,27 +630,98 @@ impl CosmosSdkChain {
     ///
     /// Returns an error if the node is still syncing and has not caught up,
     /// ie. if `sync_info.catching_up` is `true`.
-    fn chain_status(&self) -> Result<status::Response, Error> {
+    fn chain_rpc_status(&self) -> Result<status::Response, Error> {
         crate::time!(
-            "chain_status",
+            "chain_rpc_status",
             {
                 "src_chain": self.config().id.to_string(),
             }
         );
-        crate::telemetry!(query, self.id(), "status");
+        crate::telemetry!(query, self.id(), "rpc_status");
 
         let status = self
             .block_on(self.rpc_client.status())
             .map_err(|e| Error::rpc(self.config.rpc_addr.clone(), e))?;
 
         if status.sync_info.catching_up {
+            Err(Error::chain_not_caught_up(
+                self.config.rpc_addr.to_string(),
+                self.config().id.clone(),
+            ))
+        } else {
+            Ok(status)
+        }
+    }
+
+    /// Query the chain syncing status via a gRPC query.
+    ///
+    /// Returns an error if the node is still syncing and has not caught up,
+    /// ie. if `sync_info.syncing` is `true`.
+    fn chain_grpc_status(&self) -> Result<GetSyncingResponse, Error> {
+        crate::time!(
+            "chain_grpc_status",
+            {
+                "src_chain": self.config().id.to_string(),
+            }
+        );
+        crate::telemetry!(query, self.id(), "grpc_status");
+
+        let grpc_addr = self.grpc_addr.clone();
+        let grpc_addr_string = grpc_addr.to_string();
+
+        let mut client = self
+            .block_on(ServiceClient::connect(grpc_addr.clone()))
+            .map_err(Error::grpc_transport)
+            .unwrap();
+
+        let request = tonic::Request::new(GetSyncingRequest {});
+
+        let sync_info = self
+            .block_on(client.get_syncing(request))
+            .map_err(|e| Error::grpc_status(e, "get_syncing".to_string()))?
+            .into_inner();
+
+        if sync_info.syncing {
+            Err(Error::chain_not_caught_up(
+                grpc_addr_string,
+                self.config().id.clone(),
+            ))
+        } else {
+            Ok(sync_info)
+        }
+    }
+
+    /// Query the chain status of the RPC and gRPC nodes.
+    ///
+    /// Returns an error if any of the node is still syncing and has not caught up.
+    fn chain_status(&self) -> Result<status::Response, Error> {
+        crate::time!(
+            "chain_status",
+            {
+                "src_chain": self.config().id.to_string(),
+            }
+        );
+        crate::telemetry!(query, self.id(), "status");
+
+        let rpc_status = self.chain_rpc_status()?;
+
+        if rpc_status.sync_info.catching_up {
             return Err(Error::chain_not_caught_up(
                 self.config.rpc_addr.to_string(),
                 self.config().id.clone(),
             ));
         }
 
-        Ok(status)
+        let grpc_status = self.chain_grpc_status()?;
+
+        if grpc_status.syncing {
+            return Err(Error::chain_not_caught_up(
+                self.config.grpc_addr.to_string(),
+                self.config().id.clone(),
+            ));
+        }
+
+        Ok(rpc_status)
     }
 
     /// Query the chain's latest height