Fix some CI flakiness by ensuring we don't bind the same port twice o…

…n chain startup (#3422)

tools/test-framework/Cargo.toml

@@ -41,3 +41,4 @@ flex-error = "0.4.4"
 prost = { version = "0.11" }
 tonic = { version = "0.9", features = ["tls", "tls-roots"] }
 hdpath = "0.6.3"
+once_cell = "1.18.0"

tools/test-framework/src/bootstrap/single.rs

@@ -97,6 +97,7 @@ pub fn bootstrap_single_node(
         config::set_log_level(config, &log_level)?;
         config::set_rpc_port(config, chain_driver.rpc_port)?;
         config::set_p2p_port(config, chain_driver.p2p_port)?;
+        config::set_pprof_port(config, chain_driver.pprof_port)?;
         config::set_timeout_commit(config, Duration::from_secs(1))?;
         config::set_timeout_propose(config, Duration::from_secs(1))?;
         config::set_mode(config, "validator")?;

tools/test-framework/src/chain/builder.rs

@@ -105,6 +105,7 @@ impl ChainBuilder {
         let grpc_port = random_unused_tcp_port();
         let grpc_web_port = random_unused_tcp_port();
         let p2p_port = random_unused_tcp_port();
+        let pprof_port = random_unused_tcp_port();
 
         let home_path = format!("{}/{}", self.base_store_dir, chain_id);
 
@@ -118,6 +119,7 @@ impl ChainBuilder {
             grpc_port,
             grpc_web_port,
             p2p_port,
+            pprof_port,
             self.runtime.clone(),
         )?;
 

tools/test-framework/src/chain/config.rs

@@ -67,6 +67,19 @@ pub fn set_p2p_port(config: &mut Value, port: u16) -> Result<(), Error> {
     Ok(())
 }
 
+/// Set the `pprof_laddr` field in the full node config.
+pub fn set_pprof_port(config: &mut Value, port: u16) -> Result<(), Error> {
+    config
+        .as_table_mut()
+        .ok_or_else(|| eyre!("expect object"))?
+        .insert(
+            "pprof_laddr".to_string(),
+            format!("tcp://0.0.0.0:{port}").into(),
+        );
+
+    Ok(())
+}
+
 pub fn set_mempool_version(config: &mut Value, version: &str) -> Result<(), Error> {
     config
         .get_mut("mempool")

tools/test-framework/src/chain/driver.rs

@@ -87,6 +87,11 @@ pub struct ChainDriver {
     */
     pub p2p_port: u16,
 
+    /**
+       The port used for pprof. (Currently unused other than for setup)
+    */
+    pub pprof_port: u16,
+
     pub tx_config: TxConfig,
 
     pub runtime: Arc<Runtime>,
@@ -113,6 +118,7 @@ impl ChainDriver {
         grpc_port: u16,
         grpc_web_port: u16,
         p2p_port: u16,
+        pprof_port: u16,
         runtime: Arc<Runtime>,
     ) -> Result<Self, Error> {
         let tx_config = new_tx_config_for_test(
@@ -132,6 +138,7 @@ impl ChainDriver {
             grpc_port,
             grpc_web_port,
             p2p_port,
+            pprof_port,
             tx_config,
             runtime,
         })

tools/test-framework/src/util/random.rs

@@ -3,8 +3,13 @@
 */
 
 use ibc_relayer_types::applications::transfer::amount::Amount;
+use once_cell::sync::Lazy;
 use rand::Rng;
-use std::net::{Ipv4Addr, SocketAddrV4, TcpListener};
+use std::{
+    collections::HashSet,
+    net::{Ipv4Addr, SocketAddrV4, TcpListener},
+    sync::Mutex,
+};
 
 /// Generates a random `u32` value.
 pub fn random_u32() -> u32 {
@@ -48,18 +53,18 @@ pub fn random_string() -> String {
 /// Generates a random non-privileged port that is greater than 1024.
 fn random_port() -> u16 {
     let mut rng = rand::thread_rng();
-    rng.gen::<u16>()
-        .checked_add(1024)
-        .unwrap_or_else(random_port)
+    rng.gen_range(1024..=u16::MAX)
 }
 
 /// Find a random unused non-privileged TCP port.
 pub fn random_unused_tcp_port() -> u16 {
+    static ALLOCATED_PORTS: Lazy<Mutex<HashSet<u16>>> = Lazy::new(|| Mutex::new(HashSet::new()));
+
     let port = random_port();
     let loopback = Ipv4Addr::new(127, 0, 0, 1);
     let address = SocketAddrV4::new(loopback, port);
     match TcpListener::bind(address) {
-        Ok(_) => port,
-        Err(_) => random_unused_tcp_port(),
+        Ok(_) if ALLOCATED_PORTS.lock().unwrap().insert(port) => port,
+        _ => random_unused_tcp_port(),
     }
 }