From 68fc9e40fe4fc909378881cbc9372e5e5a3d9e60 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 30 Sep 2024 13:57:47 -0300 Subject: [PATCH 01/22] Update consensus config --- core/lib/config/src/configs/consensus.rs | 5 +++-- core/lib/config/src/testonly.rs | 5 +++-- core/lib/protobuf_config/src/consensus.rs | 14 +++++++++----- .../protobuf_config/src/proto/core/consensus.proto | 9 ++++++--- etc/env/consensus_config.yaml | 5 +++-- etc/env/en_consensus_config.yaml | 5 +++-- .../src/commands/external_node/prepare_configs.rs | 4 ++-- .../crates/zk_inception/src/utils/consensus.rs | 9 +++++---- 8 files changed, 34 insertions(+), 22 deletions(-) diff --git a/core/lib/config/src/configs/consensus.rs b/core/lib/config/src/configs/consensus.rs index 918d8f4adab..8df2443bbd6 100644 --- a/core/lib/config/src/configs/consensus.rs +++ b/core/lib/config/src/configs/consensus.rs @@ -115,12 +115,13 @@ impl RpcConfig { /// Config (shared between main node and external node). #[derive(Clone, Debug, PartialEq)] pub struct ConsensusConfig { + pub port: u16, /// Local socket address to listen for the incoming connections. - pub server_addr: std::net::SocketAddr, + pub server_url: std::net::SocketAddr, /// Public address of this node (should forward to `server_addr`) /// that will be advertised to peers, so that they can connect to this /// node. - pub public_addr: Host, + pub public_url: Host, /// Maximal allowed size of the payload in bytes. pub max_payload_size: usize, diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 19f34cf38f7..f54254ba65d 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -798,8 +798,9 @@ impl Distribution for EncodeDist { fn sample(&self, rng: &mut R) -> configs::consensus::ConsensusConfig { use configs::consensus::{ConsensusConfig, Host, NodePublicKey}; ConsensusConfig { - server_addr: self.sample(rng), - public_addr: Host(self.sample(rng)), + port: self.sample(rng), + server_url: self.sample(rng), + public_url: Host(self.sample(rng)), max_payload_size: self.sample(rng), max_batch_size: self.sample(rng), gossip_dynamic_inbound_limit: self.sample(rng), diff --git a/core/lib/protobuf_config/src/consensus.rs b/core/lib/protobuf_config/src/consensus.rs index 81cad437fe4..7bc994fa725 100644 --- a/core/lib/protobuf_config/src/consensus.rs +++ b/core/lib/protobuf_config/src/consensus.rs @@ -148,10 +148,13 @@ impl ProtoRepr for proto::Config { }; Ok(Self::Type { - server_addr: required(&self.server_addr) + port: required(&self.port) + .and_then(|x| Ok((*x).try_into()?)) + .context("port")?, + server_url: required(&self.server_url) .and_then(|x| Ok(x.parse()?)) - .context("server_addr")?, - public_addr: Host(required(&self.public_addr).context("public_addr")?.clone()), + .context("server_url")?, + public_url: Host(required(&self.public_url).context("public_url")?.clone()), max_payload_size, max_batch_size, gossip_dynamic_inbound_limit: required(&self.gossip_dynamic_inbound_limit) @@ -182,8 +185,9 @@ impl ProtoRepr for proto::Config { fn build(this: &Self::Type) -> Self { Self { - server_addr: Some(this.server_addr.to_string()), - public_addr: Some(this.public_addr.0.clone()), + port: Some(this.port.try_into().unwrap()), + server_url: Some(this.server_url.to_string()), + public_url: Some(this.public_url.0.clone()), max_payload_size: Some(this.max_payload_size.try_into().unwrap()), max_batch_size: Some(this.max_batch_size.try_into().unwrap()), gossip_dynamic_inbound_limit: Some( diff --git a/core/lib/protobuf_config/src/proto/core/consensus.proto b/core/lib/protobuf_config/src/proto/core/consensus.proto index 92527df739a..da85096874b 100644 --- a/core/lib/protobuf_config/src/proto/core/consensus.proto +++ b/core/lib/protobuf_config/src/proto/core/consensus.proto @@ -70,13 +70,16 @@ message Config { reserved 3; reserved "validators"; + // Port to listen on, for incoming TCP connections. + optional uint32 port = 12; // required + // IP:port to listen on, for incoming TCP connections. // Use `0.0.0.0:` to listen on all network interfaces (i.e. on all IPs exposed by this VM). - optional string server_addr = 1; // required; IpAddr + optional string server_url = 1; // required; IpAddr - // Public IP:port to advertise, should forward to server_addr. + // Public IP:port to advertise, should forward to server_url. // Can be `127.0.0.1:` for local tests. - optional string public_addr = 2; // required; Host + optional string public_url = 2; // required; Host // Maximal allowed size of the payload. optional uint64 max_payload_size = 4; // required; bytes diff --git a/etc/env/consensus_config.yaml b/etc/env/consensus_config.yaml index 304ea31fac9..411d0941df0 100644 --- a/etc/env/consensus_config.yaml +++ b/etc/env/consensus_config.yaml @@ -1,5 +1,6 @@ -server_addr: "127.0.0.1:3054" -public_addr: "127.0.0.1:3054" +port: 3054 +server_url: "127.0.0.1:3054" +public_url: "127.0.0.1:3054" max_payload_size: 2500000 gossip_dynamic_inbound_limit: 1 # LOCALHOST TEST CONFIGURATION ONLY, don't copy to other environments. diff --git a/etc/env/en_consensus_config.yaml b/etc/env/en_consensus_config.yaml index f759e72e891..874c663efe7 100644 --- a/etc/env/en_consensus_config.yaml +++ b/etc/env/en_consensus_config.yaml @@ -1,5 +1,6 @@ -server_addr: '127.0.0.1:3055' -public_addr: '127.0.0.1:3055' +port: 3055 +server_url: '127.0.0.1:3055' +public_url: '127.0.0.1:3055' max_payload_size: 2500000 gossip_dynamic_inbound_limit: 0 gossip_static_outbound: diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs index bc741a6eb38..2c0879c7c83 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -86,7 +86,7 @@ fn prepare_configs( // TODO: This is a temporary solution. We should allocate consensus port using `EcosystemPorts::allocate_ports_in_yaml` ports.add_port_info( - main_node_consensus_config.server_addr.port(), + main_node_consensus_config.server_url.port(), "Main node consensus".to_string(), ); let offset = ((config.id - 1) * 100) as u16; @@ -102,7 +102,7 @@ fn prepare_configs( )? .context(MSG_CONSENSUS_SECRETS_NODE_KEY_MISSING_ERR)?; - gossip_static_outbound.insert(main_node_public_key, main_node_consensus_config.public_addr); + gossip_static_outbound.insert(main_node_public_key, main_node_consensus_config.public_url); let en_consensus_config = get_consensus_config(config, consensus_port, None, Some(gossip_static_outbound))?; diff --git a/zk_toolbox/crates/zk_inception/src/utils/consensus.rs b/zk_toolbox/crates/zk_inception/src/utils/consensus.rs index 2979b4df0c1..fe650c0beab 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/consensus.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/consensus.rs @@ -57,12 +57,13 @@ pub fn get_consensus_config( let genesis_spec = consensus_keys.map(|consensus_keys| get_genesis_specs(chain_config, &consensus_keys)); - let public_addr = SocketAddr::new(CONSENSUS_PUBLIC_ADDRESS_HOST, consensus_port); - let server_addr = SocketAddr::new(CONSENSUS_SERVER_ADDRESS_HOST, consensus_port); + let public_url = SocketAddr::new(CONSENSUS_PUBLIC_ADDRESS_HOST, consensus_port); + let server_url = SocketAddr::new(CONSENSUS_SERVER_ADDRESS_HOST, consensus_port); Ok(ConsensusConfig { - server_addr, - public_addr: Host(public_addr.encode()), + port: consensus_port, + server_url, + public_url: Host(public_url.encode()), genesis_spec, max_payload_size: MAX_PAYLOAD_SIZE, gossip_dynamic_inbound_limit: GOSSIP_DYNAMIC_INBOUND_LIMIT, From 0d87c22098c3e40025d6f358029c778e3d17d1dc Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 30 Sep 2024 14:07:10 -0300 Subject: [PATCH 02/22] Fix executor config --- core/node/consensus/src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/node/consensus/src/config.rs b/core/node/consensus/src/config.rs index 3584d533f66..abddbb6c762 100644 --- a/core/node/consensus/src/config.rs +++ b/core/node/consensus/src/config.rs @@ -166,8 +166,8 @@ pub(super) fn executor( Ok(executor::Config { build_version, - server_addr: cfg.server_addr, - public_addr: net::Host(cfg.public_addr.0.clone()), + server_addr: cfg.server_url, + public_addr: net::Host(cfg.public_url.0.clone()), max_payload_size: cfg.max_payload_size, max_batch_size: cfg.max_batch_size, node_key: node_key(secrets) From f2adb09d81fdd204e5ec7c8467213a454cd8b950 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 30 Sep 2024 14:14:02 -0300 Subject: [PATCH 03/22] Fix make_config --- core/node/consensus/src/testonly.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/node/consensus/src/testonly.rs b/core/node/consensus/src/testonly.rs index 04a2dfbc083..d221109b61e 100644 --- a/core/node/consensus/src/testonly.rs +++ b/core/node/consensus/src/testonly.rs @@ -154,8 +154,9 @@ fn make_config( genesis_spec: Option, ) -> config::ConsensusConfig { config::ConsensusConfig { - server_addr: *cfg.server_addr, - public_addr: config::Host(cfg.public_addr.0.clone()), + port: cfg.server_addr.port(), + server_url: *cfg.server_addr, + public_url: config::Host(cfg.public_addr.0.clone()), max_payload_size: usize::MAX, max_batch_size: usize::MAX, gossip_dynamic_inbound_limit: cfg.gossip.dynamic_inbound_limit, From 328bd328f49ed82261a16e7efc72deba0c6f36dc Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 30 Sep 2024 14:26:33 -0300 Subject: [PATCH 04/22] Update proto --- core/lib/protobuf_config/src/proto/core/consensus.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/lib/protobuf_config/src/proto/core/consensus.proto b/core/lib/protobuf_config/src/proto/core/consensus.proto index da85096874b..d7231cef38a 100644 --- a/core/lib/protobuf_config/src/proto/core/consensus.proto +++ b/core/lib/protobuf_config/src/proto/core/consensus.proto @@ -75,11 +75,11 @@ message Config { // IP:port to listen on, for incoming TCP connections. // Use `0.0.0.0:` to listen on all network interfaces (i.e. on all IPs exposed by this VM). - optional string server_url = 1; // required; IpAddr + optional string server_url = 13; // required; IpAddr // Public IP:port to advertise, should forward to server_url. // Can be `127.0.0.1:` for local tests. - optional string public_url = 2; // required; Host + optional string public_url = 14; // required; Host // Maximal allowed size of the payload. optional uint64 max_payload_size = 4; // required; bytes From 73dddaba3f3785a5b34b14581ac40386198b5624 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 30 Sep 2024 14:32:15 -0300 Subject: [PATCH 05/22] Reserve fields --- core/lib/protobuf_config/src/proto/core/consensus.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/lib/protobuf_config/src/proto/core/consensus.proto b/core/lib/protobuf_config/src/proto/core/consensus.proto index d7231cef38a..1159dc990f7 100644 --- a/core/lib/protobuf_config/src/proto/core/consensus.proto +++ b/core/lib/protobuf_config/src/proto/core/consensus.proto @@ -67,6 +67,10 @@ message RpcConfig { } message Config { + reserved 1; + reserved server_addr; + reserver 2; + reserved server_addr; reserved 3; reserved "validators"; From 88b702394d81fe0e123368088d822c101c3f9606 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 30 Sep 2024 14:35:57 -0300 Subject: [PATCH 06/22] Fix reserved --- core/lib/protobuf_config/src/proto/core/consensus.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/lib/protobuf_config/src/proto/core/consensus.proto b/core/lib/protobuf_config/src/proto/core/consensus.proto index 1159dc990f7..c8a7fa17e5e 100644 --- a/core/lib/protobuf_config/src/proto/core/consensus.proto +++ b/core/lib/protobuf_config/src/proto/core/consensus.proto @@ -68,9 +68,9 @@ message RpcConfig { message Config { reserved 1; - reserved server_addr; + reserved "server_addr"; reserver 2; - reserved server_addr; + reserved "server_addr"; reserved 3; reserved "validators"; From 85f81cfcf71438694d97a6cb79664eb5dbf999b5 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 30 Sep 2024 14:36:07 -0300 Subject: [PATCH 07/22] Fix reserved --- core/lib/protobuf_config/src/proto/core/consensus.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/protobuf_config/src/proto/core/consensus.proto b/core/lib/protobuf_config/src/proto/core/consensus.proto index c8a7fa17e5e..58f839dcbe1 100644 --- a/core/lib/protobuf_config/src/proto/core/consensus.proto +++ b/core/lib/protobuf_config/src/proto/core/consensus.proto @@ -69,7 +69,7 @@ message RpcConfig { message Config { reserved 1; reserved "server_addr"; - reserver 2; + reserved 2; reserved "server_addr"; reserved 3; reserved "validators"; From c4e9c045dd4efe96c34704ae47c738f9ea6f9f77 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 30 Sep 2024 14:48:17 -0300 Subject: [PATCH 08/22] Fix duplicated reserved name --- core/lib/protobuf_config/src/proto/core/consensus.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/protobuf_config/src/proto/core/consensus.proto b/core/lib/protobuf_config/src/proto/core/consensus.proto index 58f839dcbe1..0846d2627e6 100644 --- a/core/lib/protobuf_config/src/proto/core/consensus.proto +++ b/core/lib/protobuf_config/src/proto/core/consensus.proto @@ -70,7 +70,7 @@ message Config { reserved 1; reserved "server_addr"; reserved 2; - reserved "server_addr"; + reserved "public_addr"; reserved 3; reserved "validators"; From 55b60e16fa032b3f2c9ef218c78daecd16c83fad Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 1 Oct 2024 11:03:06 -0300 Subject: [PATCH 09/22] lint --- core/lib/protobuf_config/src/consensus.rs | 2 +- zk_toolbox/crates/zk_inception/src/utils/ports.rs | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/core/lib/protobuf_config/src/consensus.rs b/core/lib/protobuf_config/src/consensus.rs index 7bc994fa725..a8bd3dde839 100644 --- a/core/lib/protobuf_config/src/consensus.rs +++ b/core/lib/protobuf_config/src/consensus.rs @@ -185,7 +185,7 @@ impl ProtoRepr for proto::Config { fn build(this: &Self::Type) -> Self { Self { - port: Some(this.port.try_into().unwrap()), + port: Some(this.port.into()), server_url: Some(this.server_url.to_string()), public_url: Some(this.public_url.0.clone()), max_payload_size: Some(this.max_payload_size.try_into().unwrap()), diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index 0462c0ea40f..9c972a3de27 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -21,10 +21,6 @@ pub struct EcosystemPorts { } impl EcosystemPorts { - pub fn get_assigned_ports(&self) -> HashSet { - self.ports.keys().cloned().collect() - } - pub fn is_port_assigned(&self, port: u16) -> bool { self.ports.contains_key(&port) } From 480a747fd98354665dfa5f8b73653e57274f3f12 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 1 Oct 2024 11:20:08 -0300 Subject: [PATCH 10/22] Remove unused import --- zk_toolbox/crates/zk_inception/src/utils/ports.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index 9c972a3de27..5102b4fd9c6 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -1,9 +1,4 @@ -use std::{ - collections::{HashMap, HashSet}, - fmt, - ops::Range, - path::Path, -}; +use std::{collections::HashMap, fmt, ops::Range, path::Path}; use anyhow::{bail, Context, Result}; use config::{ From fc8468684a17ad696bfa4f7081202b0d0046425d Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 1 Oct 2024 22:19:30 -0300 Subject: [PATCH 11/22] add consensus to general yaml --- core/node/eth_watch/src/tests.rs | 2 +- etc/env/file_based/general.yaml | 7 ++++ .../zk_inception/src/commands/chain/init.rs | 14 +++---- .../crates/zk_inception/src/utils/ports.rs | 39 +++++++++++++++---- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/core/node/eth_watch/src/tests.rs b/core/node/eth_watch/src/tests.rs index 8c37b5c9920..feb9eff35b5 100644 --- a/core/node/eth_watch/src/tests.rs +++ b/core/node/eth_watch/src/tests.rs @@ -152,7 +152,7 @@ impl EthClient for MockEthClient { Ok(logs .into_iter() .filter(|log| { - log.topics.get(0) == Some(&topic1) + log.topics.first() == Some(&topic1) && (topic2.is_none() || log.topics.get(1) == topic2.as_ref()) }) .collect()) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index a4ba8c0201a..2e8008d084d 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -375,3 +375,10 @@ da_dispatcher: external_proof_integration_api: http_port: 3073 + +consensus: + port: 3054 + server_url: "127.0.0.1:3054" + public_url: "127.0.0.1:3054" + max_payload_size: 2500000 + gossip_dynamic_inbound_limit: 100 diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 685e6165d25..7fa6bde9933 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -23,8 +23,9 @@ use crate::{ defaults::PORT_RANGE_END, messages::{ msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_INITIALIZED, - MSG_CHAIN_NOT_FOUND_ERR, MSG_DEPLOYING_PAYMASTER, MSG_GENESIS_DATABASE_ERR, - MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, + MSG_CHAIN_NOT_FOUND_ERR, MSG_CONSENSUS_CONFIG_MISSING_ERR, MSG_DEPLOYING_PAYMASTER, + MSG_GENESIS_DATABASE_ERR, MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, + MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, MSG_UPDATING_TOKEN_MULTIPLIER_SETTER_SPINNER, MSG_WALLET_TOKEN_MULTIPLIER_SETTER_NOT_FOUND, }, utils::{ @@ -68,12 +69,11 @@ pub async fn init( )?; } let mut general_config = chain_config.get_general_config()?; + let mut consensus_config = general_config + .consensus_config + .context(MSG_CONSENSUS_CONFIG_MISSING_ERR)?; - // TODO: This is a temporary solution. We should allocate consensus port using `EcosystemPorts::allocate_ports_in_yaml` - let offset = ((chain_config.id - 1) * 100) as u16; - let consensus_port_range = DEFAULT_CONSENSUS_PORT + offset..PORT_RANGE_END; - let consensus_port = - ecosystem_ports.allocate_port(consensus_port_range, "Consensus".to_string())?; + let consensus_port = consensus_config.port; let consensus_keys = generate_consensus_keys(); let consensus_config = get_consensus_config( diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index 5102b4fd9c6..5d694d4fc68 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, fmt, ops::Range, path::Path}; +use std::{collections::HashMap, fmt, net::SocketAddr, ops::Range, path::Path}; use anyhow::{bail, Context, Result}; use config::{ @@ -99,15 +99,38 @@ impl EcosystemPorts { // Update ports in URLs for (key, val) in &mut *map { if key.as_str().map(|s| s.ends_with("url")).unwrap_or(false) { - let mut url = Url::parse(val.as_str().unwrap())?; - if let Some(port) = url.port() { - if let Some(new_port) = updated_ports.get(&port) { - if let Err(()) = url.set_port(Some(*new_port)) { - bail!("Failed to update port in URL {}", url); - } else { - *val = Value::String(url.to_string()); + let value = val.as_str().unwrap(); + match Url::parse(value) { + Ok(mut url) => { + if let Some(port) = url.port() { + if let Some(new_port) = updated_ports.get(&port) { + if let Err(()) = url.set_port(Some(*new_port)) { + bail!("Failed to update port in URL {}", url); + } else { + *val = Value::String(url.to_string()); + } + } } } + Err(url::ParseError::RelativeUrlWithoutBase) => { + match value.parse::() { + Ok(socket_addr) => { + if let Some(new_port) = + updated_ports.get(&socket_addr.port()) + { + let new_socket_addr = + SocketAddr::new(socket_addr.ip(), *new_port); + *val = Value::String(new_socket_addr.to_string()); + } + } + Err(err) => { + bail!("Failed to parse socket address: {}", err); + } + } + } + Err(err) => { + bail!("Failed to parse URL: {}", err); + } } } } From f8a2103a90e5618172e15cabdb72aaf0d35a947d Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 07:55:39 -0300 Subject: [PATCH 12/22] Remove manual main node port allocation for reapre configs --- zk_toolbox/crates/zk_inception/src/commands/chain/init.rs | 5 ++--- .../src/commands/external_node/prepare_configs.rs | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 7fa6bde9933..a3546456e3a 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -2,7 +2,7 @@ use anyhow::Context; use common::{config::global_config, git, logger, spinner::Spinner}; use config::{ copy_configs, set_l1_rpc_url, traits::SaveConfigWithBasePath, update_from_chain_config, - ChainConfig, EcosystemConfig, DEFAULT_CONSENSUS_PORT, + ChainConfig, EcosystemConfig, }; use types::BaseToken; use xshell::Shell; @@ -20,7 +20,6 @@ use crate::{ }, portal::update_portal_config, }, - defaults::PORT_RANGE_END, messages::{ msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, MSG_CONSENSUS_CONFIG_MISSING_ERR, MSG_DEPLOYING_PAYMASTER, @@ -69,7 +68,7 @@ pub async fn init( )?; } let mut general_config = chain_config.get_general_config()?; - let mut consensus_config = general_config + let consensus_config = general_config .consensus_config .context(MSG_CONSENSUS_CONFIG_MISSING_ERR)?; diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs index 2c0879c7c83..50f94309349 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -85,10 +85,6 @@ fn prepare_configs( .context(MSG_CONSENSUS_CONFIG_MISSING_ERR)?; // TODO: This is a temporary solution. We should allocate consensus port using `EcosystemPorts::allocate_ports_in_yaml` - ports.add_port_info( - main_node_consensus_config.server_url.port(), - "Main node consensus".to_string(), - ); let offset = ((config.id - 1) * 100) as u16; let consensus_port_range = DEFAULT_CONSENSUS_PORT + offset..PORT_RANGE_END; let consensus_port = ports.allocate_port(consensus_port_range, "Consensus".to_string())?; From fd2902fa5ddcf7ebb373a39199998a2367843c34 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 08:22:13 -0300 Subject: [PATCH 13/22] Refactor `prepare_configs` --- .../commands/external_node/prepare_configs.rs | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs index 50f94309349..0e91cf67e73 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -11,7 +11,7 @@ use config::{ use xshell::Shell; use zksync_basic_types::url::SensitiveUrl; use zksync_config::configs::{ - consensus::{ConsensusSecrets, NodeSecretKey, Secret}, + consensus::{ConsensusConfig, ConsensusSecrets, NodeSecretKey, Secret}, DatabaseSecrets, L1Secrets, }; use zksync_consensus_crypto::TextFmt; @@ -79,16 +79,12 @@ fn prepare_configs( gateway_url: None, }; let mut general_en = general.clone(); + general_en.consensus_config = None; let main_node_consensus_config = general .consensus_config .context(MSG_CONSENSUS_CONFIG_MISSING_ERR)?; - // TODO: This is a temporary solution. We should allocate consensus port using `EcosystemPorts::allocate_ports_in_yaml` - let offset = ((config.id - 1) * 100) as u16; - let consensus_port_range = DEFAULT_CONSENSUS_PORT + offset..PORT_RANGE_END; - let consensus_port = ports.allocate_port(consensus_port_range, "Consensus".to_string())?; - let mut gossip_static_outbound = BTreeMap::new(); let main_node_public_key = node_public_key( &config @@ -100,10 +96,11 @@ fn prepare_configs( gossip_static_outbound.insert(main_node_public_key, main_node_consensus_config.public_url); + // Set a default port for the external node + // This is allocated later with `EcosystemPorts::allocate_ports_in_yaml` + let consensus_port = main_node_consensus_config.port; let en_consensus_config = get_consensus_config(config, consensus_port, None, Some(gossip_static_outbound))?; - general_en.consensus_config = Some(en_consensus_config.clone()); - en_consensus_config.save_with_base_path(shell, en_configs_path)?; // Set secrets config let node_key = roles::node::SecretKey::generate().encode(); @@ -124,16 +121,25 @@ fn prepare_configs( }), data_availability: None, }; - secrets.save_with_base_path(shell, en_configs_path)?; + let dirs = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::ExternalNode)?; set_rocks_db_config(&mut general_en, dirs)?; + general_en.save_with_base_path(shell, en_configs_path)?; en_config.save_with_base_path(shell, en_configs_path)?; + en_consensus_config.save_with_base_path(shell, en_configs_path)?; + secrets.save_with_base_path(shell, en_configs_path)?; + let offset = 0; // This is zero because general_en ports already have a chain offset ports.allocate_ports_in_yaml( shell, &GeneralConfig::get_path_with_base_path(en_configs_path), - 0, // This is zero because general_en ports already have a chain offset + offset, + )?; + ports.allocate_ports_in_yaml( + shell, + &ConsensusConfig::get_path_with_base_path(en_configs_path), + offset, )?; Ok(()) From d34a5461f0232d47e49b42930926f518978395a2 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 08:29:22 -0300 Subject: [PATCH 14/22] Remove DEFAULT_CONSENSUS_PORT --- zk_toolbox/crates/config/src/consts.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 80b204cc619..f462ce33b8f 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -62,8 +62,6 @@ pub const DEFAULT_EXPLORER_WORKER_PORT: u16 = 3001; pub const DEFAULT_EXPLORER_API_PORT: u16 = 3002; /// Default port for the explorer data fetcher service pub const DEFAULT_EXPLORER_DATA_FETCHER_PORT: u16 = 3040; -/// Default port for consensus service -pub const DEFAULT_CONSENSUS_PORT: u16 = 3054; pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api"; pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher"; From 2c2a6580356b0965273976bf3a32d1a259e933b3 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 09:20:28 -0300 Subject: [PATCH 15/22] Use get_genesis_specs in chain init --- .../crates/zk_inception/src/commands/chain/init.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 9736e5afc57..63e72b97b54 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -28,7 +28,7 @@ use crate::{ MSG_UPDATING_TOKEN_MULTIPLIER_SETTER_SPINNER, MSG_WALLET_TOKEN_MULTIPLIER_SETTER_NOT_FOUND, }, utils::{ - consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets}, + consensus::{generate_consensus_keys, get_consensus_secrets, get_genesis_specs}, ports::EcosystemPortsScanner, }, }; @@ -67,19 +67,13 @@ pub async fn init( )?; } let mut general_config = chain_config.get_general_config()?; - let consensus_config = general_config + let mut consensus_config = general_config .consensus_config .context(MSG_CONSENSUS_CONFIG_MISSING_ERR)?; - let consensus_port = consensus_config.port; - let consensus_keys = generate_consensus_keys(); - let consensus_config = get_consensus_config( - chain_config, - consensus_port, - Some(consensus_keys.clone()), - None, - )?; + consensus_config.genesis_spec = Some(get_genesis_specs(chain_config, &consensus_keys)); + general_config.consensus_config = Some(consensus_config); general_config.save_with_base_path(shell, &chain_config.configs)?; From 044c356b14fac4e29e7ccb5b7d6b7f05a5d4e6b4 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 10:01:49 -0300 Subject: [PATCH 16/22] Remove `get_consensus_config` --- .../commands/external_node/prepare_configs.rs | 11 ++--- zk_toolbox/crates/zk_inception/src/consts.rs | 23 ---------- .../zk_inception/src/utils/consensus.rs | 43 ++----------------- 3 files changed, 6 insertions(+), 71 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs index 5b1d98021f3..0d8428fa2c2 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -25,7 +25,7 @@ use crate::{ MSG_CONSENSUS_SECRETS_NODE_KEY_MISSING_ERR, MSG_PREPARING_EN_CONFIGS, }, utils::{ - consensus::{get_consensus_config, node_public_key}, + consensus::node_public_key, ports::EcosystemPortsScanner, rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }, @@ -82,6 +82,7 @@ fn prepare_configs( let main_node_consensus_config = general .consensus_config .context(MSG_CONSENSUS_CONFIG_MISSING_ERR)?; + let mut en_consensus_config = main_node_consensus_config.clone(); let mut gossip_static_outbound = BTreeMap::new(); let main_node_public_key = node_public_key( @@ -91,14 +92,8 @@ fn prepare_configs( .context(MSG_CONSENSUS_SECRETS_MISSING_ERR)?, )? .context(MSG_CONSENSUS_SECRETS_NODE_KEY_MISSING_ERR)?; - gossip_static_outbound.insert(main_node_public_key, main_node_consensus_config.public_url); - - // Set a default port for the external node - // This is allocated later with `EcosystemPorts::allocate_ports_in_yaml` - let consensus_port = main_node_consensus_config.port; - let en_consensus_config = - get_consensus_config(config, consensus_port, None, Some(gossip_static_outbound))?; + en_consensus_config.gossip_static_outbound = gossip_static_outbound; // Set secrets config let node_key = roles::node::SecretKey::generate().encode(); diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index 9f81847e333..df27d2f02d2 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -1,5 +1,3 @@ -use std::net::{IpAddr, Ipv4Addr}; - pub const AMOUNT_FOR_DISTRIBUTION_TO_WALLETS: u128 = 1000000000000000000000; pub const MINIMUM_BALANCE_FOR_WALLET: u128 = 5000000000000000000; @@ -12,27 +10,6 @@ pub const DEFAULT_UNSIGNED_TRANSACTIONS_DIR: &str = "transactions"; pub const BELLMAN_CUDA_DIR: &str = "era-bellman-cuda"; pub const L2_BASE_TOKEN_ADDRESS: &str = "0x000000000000000000000000000000000000800A"; -#[allow(non_upper_case_globals)] -const kB: usize = 1024; - -/// Max payload size for consensus in bytes -pub const MAX_PAYLOAD_SIZE: usize = 2_500_000; -/// Max batch size for consensus in bytes -/// Compute a default batch size, so operators are not caught out by the missing setting -/// while we're still working on batch syncing. The batch interval is ~1 minute, -/// so there will be ~60 blocks, and an Ethereum Merkle proof is ~1kB, but under high -/// traffic there can be thousands of huge transactions that quickly fill up blocks -/// and there could be more blocks in a batch then expected. We chose a generous -/// limit so as not to prevent any legitimate batch from being transmitted. -pub const MAX_BATCH_SIZE: usize = MAX_PAYLOAD_SIZE * 5000 + kB; -/// Gossip dynamic inbound limit for consensus -pub const GOSSIP_DYNAMIC_INBOUND_LIMIT: usize = 100; - -/// Public address for consensus -pub const CONSENSUS_PUBLIC_ADDRESS_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED); -/// Server address for consensus -pub const CONSENSUS_SERVER_ADDRESS_HOST: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST); - /// Path to the JS runtime config for the block-explorer-app docker container to be mounted to pub const EXPLORER_APP_DOCKER_CONFIG_PATH: &str = "/usr/src/app/packages/app/dist/config.js"; pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; diff --git a/zk_toolbox/crates/zk_inception/src/utils/consensus.rs b/zk_toolbox/crates/zk_inception/src/utils/consensus.rs index fe650c0beab..946d28a33fb 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/consensus.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/consensus.rs @@ -1,24 +1,14 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - net::SocketAddr, -}; - use anyhow::Context as _; use config::ChainConfig; use secrecy::{ExposeSecret, Secret}; use zksync_config::configs::consensus::{ - AttesterPublicKey, AttesterSecretKey, ConsensusConfig, ConsensusSecrets, GenesisSpec, Host, - NodePublicKey, NodeSecretKey, ProtocolVersion, ValidatorPublicKey, ValidatorSecretKey, - WeightedAttester, WeightedValidator, + AttesterPublicKey, AttesterSecretKey, ConsensusSecrets, GenesisSpec, NodePublicKey, + NodeSecretKey, ProtocolVersion, ValidatorPublicKey, ValidatorSecretKey, WeightedAttester, + WeightedValidator, }; use zksync_consensus_crypto::{Text, TextFmt}; use zksync_consensus_roles::{attester, node, validator}; -use crate::consts::{ - CONSENSUS_PUBLIC_ADDRESS_HOST, CONSENSUS_SERVER_ADDRESS_HOST, GOSSIP_DYNAMIC_INBOUND_LIMIT, - MAX_BATCH_SIZE, MAX_PAYLOAD_SIZE, -}; - pub(crate) fn parse_attester_committee( attesters: &[WeightedAttester], ) -> anyhow::Result { @@ -48,33 +38,6 @@ pub struct ConsensusPublicKeys { attester_key: attester::PublicKey, } -pub fn get_consensus_config( - chain_config: &ChainConfig, - consensus_port: u16, - consensus_keys: Option, - gossip_static_outbound: Option>, -) -> anyhow::Result { - let genesis_spec = - consensus_keys.map(|consensus_keys| get_genesis_specs(chain_config, &consensus_keys)); - - let public_url = SocketAddr::new(CONSENSUS_PUBLIC_ADDRESS_HOST, consensus_port); - let server_url = SocketAddr::new(CONSENSUS_SERVER_ADDRESS_HOST, consensus_port); - - Ok(ConsensusConfig { - port: consensus_port, - server_url, - public_url: Host(public_url.encode()), - genesis_spec, - max_payload_size: MAX_PAYLOAD_SIZE, - gossip_dynamic_inbound_limit: GOSSIP_DYNAMIC_INBOUND_LIMIT, - max_batch_size: MAX_BATCH_SIZE, - gossip_static_inbound: BTreeSet::new(), - gossip_static_outbound: gossip_static_outbound.unwrap_or_default(), - rpc: None, - debug_page_addr: None, - }) -} - pub fn generate_consensus_keys() -> ConsensusSecretKeys { ConsensusSecretKeys { validator_key: validator::SecretKey::generate(), From 8b93be30678a5cbe7b9d7ec6e2ca33daa18a88d3 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 10:52:07 -0300 Subject: [PATCH 17/22] Make it backward compatible --- core/lib/protobuf_config/src/consensus.rs | 22 +++++++++++++++---- .../src/proto/core/consensus.proto | 10 +++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/core/lib/protobuf_config/src/consensus.rs b/core/lib/protobuf_config/src/consensus.rs index a8bd3dde839..46a95af8c6e 100644 --- a/core/lib/protobuf_config/src/consensus.rs +++ b/core/lib/protobuf_config/src/consensus.rs @@ -147,14 +147,26 @@ impl ProtoRepr for proto::Config { } }; + let server_url = match required(&self.server_url) { + Ok(x) => x.parse().context("server_url")?, + Err(_) => required(&self.server_addr) + .and_then(|x| Ok(x.parse()?)) + .context("server_url")?, + }; + + let public_url = match required(&self.public_url) { + Ok(x) => Host(x.clone()), + Err(_) => required(&self.public_addr) + .and_then(|x| Ok(Host(x.clone()))) + .context("public_url")?, + }; + Ok(Self::Type { port: required(&self.port) .and_then(|x| Ok((*x).try_into()?)) .context("port")?, - server_url: required(&self.server_url) - .and_then(|x| Ok(x.parse()?)) - .context("server_url")?, - public_url: Host(required(&self.public_url).context("public_url")?.clone()), + server_url, + public_url, max_payload_size, max_batch_size, gossip_dynamic_inbound_limit: required(&self.gossip_dynamic_inbound_limit) @@ -188,6 +200,8 @@ impl ProtoRepr for proto::Config { port: Some(this.port.into()), server_url: Some(this.server_url.to_string()), public_url: Some(this.public_url.0.clone()), + server_addr: None, + public_addr: None, max_payload_size: Some(this.max_payload_size.try_into().unwrap()), max_batch_size: Some(this.max_batch_size.try_into().unwrap()), gossip_dynamic_inbound_limit: Some( diff --git a/core/lib/protobuf_config/src/proto/core/consensus.proto b/core/lib/protobuf_config/src/proto/core/consensus.proto index 0846d2627e6..3bdfe50618f 100644 --- a/core/lib/protobuf_config/src/proto/core/consensus.proto +++ b/core/lib/protobuf_config/src/proto/core/consensus.proto @@ -67,10 +67,6 @@ message RpcConfig { } message Config { - reserved 1; - reserved "server_addr"; - reserved 2; - reserved "public_addr"; reserved 3; reserved "validators"; @@ -85,6 +81,12 @@ message Config { // Can be `127.0.0.1:` for local tests. optional string public_url = 14; // required; Host + // deprecated - use server_url instead + optional string server_addr = 1; + + // deprecated - use public_url instead + optional string public_addr = 2; + // Maximal allowed size of the payload. optional uint64 max_payload_size = 4; // required; bytes From ff2ca1d6a11d87f312ac78b98cf397bfdc4cc4ed Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 10:55:43 -0300 Subject: [PATCH 18/22] Update consensus deserialize --- core/lib/protobuf_config/src/consensus.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/lib/protobuf_config/src/consensus.rs b/core/lib/protobuf_config/src/consensus.rs index 46a95af8c6e..36bded3fa4f 100644 --- a/core/lib/protobuf_config/src/consensus.rs +++ b/core/lib/protobuf_config/src/consensus.rs @@ -147,16 +147,16 @@ impl ProtoRepr for proto::Config { } }; - let server_url = match required(&self.server_url) { - Ok(x) => x.parse().context("server_url")?, - Err(_) => required(&self.server_addr) + let server_url = match &self.server_url { + Some(x) => x.parse().context("server_url")?, + None => required(&self.server_addr) .and_then(|x| Ok(x.parse()?)) .context("server_url")?, }; - let public_url = match required(&self.public_url) { - Ok(x) => Host(x.clone()), - Err(_) => required(&self.public_addr) + let public_url = match &self.public_url { + Some(x) => Host(x.clone()), + None => required(&self.public_addr) .and_then(|x| Ok(Host(x.clone()))) .context("public_url")?, }; From 626cf528d6593faf6375f83097823939d9c644fe Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 13:56:12 -0300 Subject: [PATCH 19/22] Rollback url rename --- core/lib/config/src/configs/consensus.rs | 4 +-- core/lib/config/src/testonly.rs | 4 +-- core/lib/protobuf_config/src/consensus.rs | 26 ++++------------ .../src/proto/core/consensus.proto | 12 ++------ etc/env/consensus_config.yaml | 4 +-- etc/env/en_consensus_config.yaml | 4 +-- etc/env/file_based/general.yaml | 4 +-- .../commands/external_node/prepare_configs.rs | 2 +- .../crates/zk_inception/src/utils/ports.rs | 30 ++++++++----------- 9 files changed, 33 insertions(+), 57 deletions(-) diff --git a/core/lib/config/src/configs/consensus.rs b/core/lib/config/src/configs/consensus.rs index 8df2443bbd6..d864d5d44da 100644 --- a/core/lib/config/src/configs/consensus.rs +++ b/core/lib/config/src/configs/consensus.rs @@ -117,11 +117,11 @@ impl RpcConfig { pub struct ConsensusConfig { pub port: u16, /// Local socket address to listen for the incoming connections. - pub server_url: std::net::SocketAddr, + pub server_addr: std::net::SocketAddr, /// Public address of this node (should forward to `server_addr`) /// that will be advertised to peers, so that they can connect to this /// node. - pub public_url: Host, + pub public_addr: Host, /// Maximal allowed size of the payload in bytes. pub max_payload_size: usize, diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 3df60d4c42b..92f220bfb9a 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -800,8 +800,8 @@ impl Distribution for EncodeDist { use configs::consensus::{ConsensusConfig, Host, NodePublicKey}; ConsensusConfig { port: self.sample(rng), - server_url: self.sample(rng), - public_url: Host(self.sample(rng)), + server_addr: self.sample(rng), + public_addr: Host(self.sample(rng)), max_payload_size: self.sample(rng), max_batch_size: self.sample(rng), gossip_dynamic_inbound_limit: self.sample(rng), diff --git a/core/lib/protobuf_config/src/consensus.rs b/core/lib/protobuf_config/src/consensus.rs index 36bded3fa4f..37f0c52b7aa 100644 --- a/core/lib/protobuf_config/src/consensus.rs +++ b/core/lib/protobuf_config/src/consensus.rs @@ -147,26 +147,14 @@ impl ProtoRepr for proto::Config { } }; - let server_url = match &self.server_url { - Some(x) => x.parse().context("server_url")?, - None => required(&self.server_addr) - .and_then(|x| Ok(x.parse()?)) - .context("server_url")?, - }; - - let public_url = match &self.public_url { - Some(x) => Host(x.clone()), - None => required(&self.public_addr) - .and_then(|x| Ok(Host(x.clone()))) - .context("public_url")?, - }; - Ok(Self::Type { port: required(&self.port) .and_then(|x| Ok((*x).try_into()?)) .context("port")?, - server_url, - public_url, + server_addr: required(&self.server_addr) + .and_then(|x| Ok(x.parse()?)) + .context("server_addr")?, + public_addr: Host(required(&self.public_addr).context("public_addr")?.clone()), max_payload_size, max_batch_size, gossip_dynamic_inbound_limit: required(&self.gossip_dynamic_inbound_limit) @@ -198,10 +186,8 @@ impl ProtoRepr for proto::Config { fn build(this: &Self::Type) -> Self { Self { port: Some(this.port.into()), - server_url: Some(this.server_url.to_string()), - public_url: Some(this.public_url.0.clone()), - server_addr: None, - public_addr: None, + server_addr: Some(this.server_addr.to_string()), + public_addr: Some(this.public_addr.0.clone()), max_payload_size: Some(this.max_payload_size.try_into().unwrap()), max_batch_size: Some(this.max_batch_size.try_into().unwrap()), gossip_dynamic_inbound_limit: Some( diff --git a/core/lib/protobuf_config/src/proto/core/consensus.proto b/core/lib/protobuf_config/src/proto/core/consensus.proto index 3bdfe50618f..98b43f37f48 100644 --- a/core/lib/protobuf_config/src/proto/core/consensus.proto +++ b/core/lib/protobuf_config/src/proto/core/consensus.proto @@ -75,17 +75,11 @@ message Config { // IP:port to listen on, for incoming TCP connections. // Use `0.0.0.0:` to listen on all network interfaces (i.e. on all IPs exposed by this VM). - optional string server_url = 13; // required; IpAddr + optional string server_addr = 1; // required; IpAddr - // Public IP:port to advertise, should forward to server_url. + // Public IP:port to advertise, should forward to server_addr. // Can be `127.0.0.1:` for local tests. - optional string public_url = 14; // required; Host - - // deprecated - use server_url instead - optional string server_addr = 1; - - // deprecated - use public_url instead - optional string public_addr = 2; + optional string public_addr = 2; // required; Host // Maximal allowed size of the payload. optional uint64 max_payload_size = 4; // required; bytes diff --git a/etc/env/consensus_config.yaml b/etc/env/consensus_config.yaml index 411d0941df0..2564865eeb3 100644 --- a/etc/env/consensus_config.yaml +++ b/etc/env/consensus_config.yaml @@ -1,6 +1,6 @@ port: 3054 -server_url: "127.0.0.1:3054" -public_url: "127.0.0.1:3054" +server_addr: "127.0.0.1:3054" +public_addr: "127.0.0.1:3054" max_payload_size: 2500000 gossip_dynamic_inbound_limit: 1 # LOCALHOST TEST CONFIGURATION ONLY, don't copy to other environments. diff --git a/etc/env/en_consensus_config.yaml b/etc/env/en_consensus_config.yaml index 874c663efe7..5c428866cb6 100644 --- a/etc/env/en_consensus_config.yaml +++ b/etc/env/en_consensus_config.yaml @@ -1,6 +1,6 @@ port: 3055 -server_url: '127.0.0.1:3055' -public_url: '127.0.0.1:3055' +server_addr: '127.0.0.1:3055' +public_addr: '127.0.0.1:3055' max_payload_size: 2500000 gossip_dynamic_inbound_limit: 0 gossip_static_outbound: diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 2e8008d084d..017d79dbe73 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -378,7 +378,7 @@ external_proof_integration_api: consensus: port: 3054 - server_url: "127.0.0.1:3054" - public_url: "127.0.0.1:3054" + server_addr: "127.0.0.1:3054" + public_addr: "127.0.0.1:3054" max_payload_size: 2500000 gossip_dynamic_inbound_limit: 100 diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs index 0d8428fa2c2..9e698d9d4e1 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -92,7 +92,7 @@ fn prepare_configs( .context(MSG_CONSENSUS_SECRETS_MISSING_ERR)?, )? .context(MSG_CONSENSUS_SECRETS_NODE_KEY_MISSING_ERR)?; - gossip_static_outbound.insert(main_node_public_key, main_node_consensus_config.public_url); + gossip_static_outbound.insert(main_node_public_key, main_node_consensus_config.public_addr); en_consensus_config.gossip_static_outbound = gossip_static_outbound; // Set secrets config diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index 5d694d4fc68..eedf2c38002 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -99,8 +99,7 @@ impl EcosystemPorts { // Update ports in URLs for (key, val) in &mut *map { if key.as_str().map(|s| s.ends_with("url")).unwrap_or(false) { - let value = val.as_str().unwrap(); - match Url::parse(value) { + match Url::parse(val.as_str().unwrap()) { Ok(mut url) => { if let Some(port) = url.port() { if let Some(new_port) = updated_ports.get(&port) { @@ -112,24 +111,21 @@ impl EcosystemPorts { } } } - Err(url::ParseError::RelativeUrlWithoutBase) => { - match value.parse::() { - Ok(socket_addr) => { - if let Some(new_port) = - updated_ports.get(&socket_addr.port()) - { - let new_socket_addr = - SocketAddr::new(socket_addr.ip(), *new_port); - *val = Value::String(new_socket_addr.to_string()); - } - } - Err(err) => { - bail!("Failed to parse socket address: {}", err); - } + Err(err) => { + bail!("Failed to parse URL: {}", err); + } + } + } else if key.as_str().map(|s| s.ends_with("addr")).unwrap_or(false) { + match val.as_str().unwrap().parse::() { + Ok(socket_addr) => { + if let Some(new_port) = updated_ports.get(&socket_addr.port()) { + let new_socket_addr = + SocketAddr::new(socket_addr.ip(), *new_port); + *val = Value::String(new_socket_addr.to_string()); } } Err(err) => { - bail!("Failed to parse URL: {}", err); + bail!("Failed to parse socket address: {}", err); } } } From aedfa5449fd898e856eb68d103ebf7bd12e77872 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 14:09:06 -0300 Subject: [PATCH 20/22] Fix executor --- core/node/consensus/src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/node/consensus/src/config.rs b/core/node/consensus/src/config.rs index abddbb6c762..3584d533f66 100644 --- a/core/node/consensus/src/config.rs +++ b/core/node/consensus/src/config.rs @@ -166,8 +166,8 @@ pub(super) fn executor( Ok(executor::Config { build_version, - server_addr: cfg.server_url, - public_addr: net::Host(cfg.public_url.0.clone()), + server_addr: cfg.server_addr, + public_addr: net::Host(cfg.public_addr.0.clone()), max_payload_size: cfg.max_payload_size, max_batch_size: cfg.max_batch_size, node_key: node_key(secrets) From db3b7e01daf87153993d3e7ef13dca795735cb23 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 14:36:29 -0300 Subject: [PATCH 21/22] Fix testonly --- core/node/consensus/src/testonly.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/node/consensus/src/testonly.rs b/core/node/consensus/src/testonly.rs index d221109b61e..2cd315ce063 100644 --- a/core/node/consensus/src/testonly.rs +++ b/core/node/consensus/src/testonly.rs @@ -155,8 +155,8 @@ fn make_config( ) -> config::ConsensusConfig { config::ConsensusConfig { port: cfg.server_addr.port(), - server_url: *cfg.server_addr, - public_url: config::Host(cfg.public_addr.0.clone()), + server_addr: *cfg.server_addr, + public_addr: config::Host(cfg.public_addr.0.clone()), max_payload_size: usize::MAX, max_batch_size: usize::MAX, gossip_dynamic_inbound_limit: cfg.gossip.dynamic_inbound_limit, From 368af7daa34dadd6c8352e3470ad92cb7e194c2d Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 2 Oct 2024 15:05:24 -0300 Subject: [PATCH 22/22] Remove unnecessary match --- .../crates/zk_inception/src/utils/ports.rs | 35 ++++++------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index eedf2c38002..3b7b7ae7072 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -99,34 +99,21 @@ impl EcosystemPorts { // Update ports in URLs for (key, val) in &mut *map { if key.as_str().map(|s| s.ends_with("url")).unwrap_or(false) { - match Url::parse(val.as_str().unwrap()) { - Ok(mut url) => { - if let Some(port) = url.port() { - if let Some(new_port) = updated_ports.get(&port) { - if let Err(()) = url.set_port(Some(*new_port)) { - bail!("Failed to update port in URL {}", url); - } else { - *val = Value::String(url.to_string()); - } - } + let mut url = Url::parse(val.as_str().unwrap())?; + if let Some(port) = url.port() { + if let Some(new_port) = updated_ports.get(&port) { + if let Err(()) = url.set_port(Some(*new_port)) { + bail!("Failed to update port in URL {}", url); + } else { + *val = Value::String(url.to_string()); } } - Err(err) => { - bail!("Failed to parse URL: {}", err); - } } } else if key.as_str().map(|s| s.ends_with("addr")).unwrap_or(false) { - match val.as_str().unwrap().parse::() { - Ok(socket_addr) => { - if let Some(new_port) = updated_ports.get(&socket_addr.port()) { - let new_socket_addr = - SocketAddr::new(socket_addr.ip(), *new_port); - *val = Value::String(new_socket_addr.to_string()); - } - } - Err(err) => { - bail!("Failed to parse socket address: {}", err); - } + let socket_addr = val.as_str().unwrap().parse::()?; + if let Some(new_port) = updated_ports.get(&socket_addr.port()) { + let new_socket_addr = SocketAddr::new(socket_addr.ip(), *new_port); + *val = Value::String(new_socket_addr.to_string()); } } }