Skip to content

Commit

Permalink
Refactor node configs to remove local-only addresses from genesis
Browse files Browse the repository at this point in the history 
This removes the notion of internal address from the narwhal side,
instead configuring that externally.
  • Loading branch information
@aschran
aschran committed Dec 10, 2022
1 parent 1781a07 commit d7d8bf8
Show file tree
Hide file tree
Showing 21 changed files with 630 additions and 210 deletions.
117 changes: 70 additions & 47 deletions crates/sui-config/src/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@

use crate::{
genesis,
genesis_config::{GenesisConfig, ValidatorGenesisInfo},
genesis_config::{GenesisConfig, ValidatorConfigInfo, ValidatorGenesisInfo},
p2p::P2pConfig,
utils, ConsensusConfig, NetworkConfig, NodeConfig, ValidatorInfo, AUTHORITIES_DB_NAME,
CONSENSUS_DB_NAME,
};
use fastcrypto::encoding::{Encoding, Hex};
use multiaddr::Multiaddr;
use rand::rngs::OsRng;
use std::{
num::NonZeroUsize,
Expand All @@ -23,7 +24,7 @@ use sui_types::crypto::{

pub enum CommitteeConfig {
Size(NonZeroUsize),
Validators(Vec<ValidatorGenesisInfo>),
Validators(Vec<ValidatorConfigInfo>),
}

enum ValidatorIpSelection {
Expand Down Expand Up @@ -82,7 +83,7 @@ impl<R> ConfigBuilder<R> {
self
}

pub fn with_validators(mut self, validators: Vec<ValidatorGenesisInfo>) -> Self {
pub fn with_validators(mut self, validators: Vec<ValidatorConfigInfo>) -> Self {
self.committee = Some(CommitteeConfig::Validators(validators));
self
}
Expand Down Expand Up @@ -155,14 +156,19 @@ impl<R: rand::RngCore + rand::CryptoRng> ConfigBuilder<R> {
worker_key_pair: NetworkKeyPair,
account_key_pair: SuiKeyPair,
network_key_pair: NetworkKeyPair,
) -> ValidatorGenesisInfo {
) -> ValidatorConfigInfo {
match self.validator_ip_sel {
ValidatorIpSelection::Localhost => ValidatorGenesisInfo::from_localhost_for_testing(
key_pair,
worker_key_pair,
account_key_pair,
network_key_pair,
),
ValidatorIpSelection::Localhost => ValidatorConfigInfo {
genesis_info: ValidatorGenesisInfo::from_localhost_for_testing(
key_pair,
worker_key_pair,
account_key_pair,
network_key_pair,
),
consensus_address: utils::new_tcp_network_address(),
consensus_internal_worker_address: None,
},

ValidatorIpSelection::Simulator => {
// we will probably never run this many validators in a sim
let low_octet = index + 1;
Expand All @@ -171,38 +177,48 @@ impl<R: rand::RngCore + rand::CryptoRng> ConfigBuilder<R> {
}

let ip = format!("10.10.0.{}", low_octet);
let make_tcp_addr = |port: u16| -> Multiaddr {
format!("/ip4/{}/tcp/{}/http", ip, port).parse().unwrap()
};

ValidatorGenesisInfo::from_base_ip(
key_pair,
worker_key_pair,
account_key_pair,
network_key_pair,
ip,
index,
)
ValidatorConfigInfo {
genesis_info: ValidatorGenesisInfo::from_base_ip(
key_pair,
worker_key_pair,
account_key_pair,
network_key_pair,
ip.clone(),
index,
),
consensus_address: make_tcp_addr(4000 + index as u16),
consensus_internal_worker_address: None,
}
}
}
}

fn build_with_validators(
self,
mut rng: R,
validators: Vec<ValidatorGenesisInfo>,
validators: Vec<ValidatorConfigInfo>,
) -> NetworkConfig {
let validator_set = validators
.iter()
.enumerate()
.map(|(i, validator)| {
let name = format!("validator-{i}");
let protocol_key: AuthorityPublicKeyBytes = validator.key_pair.public().into();
let account_key: PublicKey = validator.account_key_pair.public();
let network_key: NetworkPublicKey = validator.network_key_pair.public().clone();
let worker_key: NetworkPublicKey = validator.worker_key_pair.public().clone();
let stake = validator.stake;
let network_address = validator.network_address.clone();
let protocol_key: AuthorityPublicKeyBytes =
validator.genesis_info.key_pair.public().into();
let account_key: PublicKey = validator.genesis_info.account_key_pair.public();
let network_key: NetworkPublicKey =
validator.genesis_info.network_key_pair.public().clone();
let worker_key: NetworkPublicKey =
validator.genesis_info.worker_key_pair.public().clone();
let stake = validator.genesis_info.stake;
let network_address = validator.genesis_info.network_address.clone();
let pop = generate_proof_of_possession(
&validator.key_pair,
(&validator.account_key_pair.public()).into(),
&validator.genesis_info.key_pair,
(&validator.genesis_info.account_key_pair.public()).into(),
);

(
Expand All @@ -214,16 +230,18 @@ impl<R: rand::RngCore + rand::CryptoRng> ConfigBuilder<R> {
account_key,
stake,
delegation: 0, // no delegation yet at genesis
gas_price: validator.gas_price,
commission_rate: validator.commission_rate,
gas_price: validator.genesis_info.gas_price,
commission_rate: validator.genesis_info.commission_rate,
network_address,
p2p_address: validator.p2p_address.clone(),
narwhal_primary_address: validator.narwhal_primary_address.clone(),
narwhal_worker_address: validator.narwhal_worker_address.clone(),
narwhal_internal_worker_address: validator
.narwhal_internal_worker_address
p2p_address: validator.genesis_info.p2p_address.clone(),
narwhal_primary_address: validator
.genesis_info
.narwhal_primary_address
.clone(),
narwhal_worker_address: validator
.genesis_info
.narwhal_worker_address
.clone(),
narwhal_consensus_address: validator.narwhal_consensus_address.clone(),
},
pop,
)
Expand All @@ -250,36 +268,41 @@ impl<R: rand::RngCore + rand::CryptoRng> ConfigBuilder<R> {
let validator_configs = validators
.into_iter()
.map(|validator| {
let public_key: AuthorityPublicKeyBytes = validator.key_pair.public().into();
let public_key: AuthorityPublicKeyBytes =
validator.genesis_info.key_pair.public().into();
let db_path = self
.config_directory
.join(AUTHORITIES_DB_NAME)
.join(Hex::encode(public_key));
let network_address = validator.network_address;
let consensus_address = validator.narwhal_consensus_address;
let network_address = validator.genesis_info.network_address;
let consensus_address = validator.consensus_address;
let consensus_db_path = self
.config_directory
.join(CONSENSUS_DB_NAME)
.join(Hex::encode(public_key));
let internal_worker_address = validator.consensus_internal_worker_address;
let consensus_config = ConsensusConfig {
consensus_address,
consensus_db_path,
address: consensus_address,
db_path: consensus_db_path,
internal_worker_address,
timeout_secs: Some(60),
narwhal_config: Default::default(),
};

let p2p_config = P2pConfig {
listen_address: utils::udp_multiaddr_to_listen_address(&validator.p2p_address)
.unwrap(),
external_address: Some(validator.p2p_address),
listen_address: utils::udp_multiaddr_to_listen_address(
&validator.genesis_info.p2p_address,
)
.unwrap(),
external_address: Some(validator.genesis_info.p2p_address),
..Default::default()
};

NodeConfig {
protocol_key_pair: Arc::new(validator.key_pair),
worker_key_pair: Arc::new(validator.worker_key_pair),
account_key_pair: Arc::new(validator.account_key_pair),
network_key_pair: Arc::new(validator.network_key_pair),
protocol_key_pair: Arc::new(validator.genesis_info.key_pair),
worker_key_pair: Arc::new(validator.genesis_info.worker_key_pair),
account_key_pair: Arc::new(validator.genesis_info.account_key_pair),
network_key_pair: Arc::new(validator.genesis_info.network_key_pair),
db_path,
network_address,
metrics_address: utils::available_local_socket_address(),
Expand Down
38 changes: 1 addition & 37 deletions crates/sui-config/src/genesis.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ use move_binary_format::CompiledModule;
use move_core_types::ident_str;
use move_core_types::language_storage::ModuleId;
use move_vm_runtime::native_functions::NativeFunctionTable;
use narwhal_crypto::NetworkPublicKey;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_with::serde_as;
use std::collections::BTreeMap;
Expand All @@ -21,8 +20,7 @@ use sui_adapter::{adapter, execution_mode};
use sui_types::base_types::ObjectID;
use sui_types::base_types::TransactionDigest;
use sui_types::chain_id::ChainId;
use sui_types::crypto::{AuthorityPublicKey, ToFromBytes};
use sui_types::crypto::{AuthorityPublicKeyBytes, AuthoritySignature};
use sui_types::crypto::{AuthorityPublicKeyBytes, AuthoritySignature, ToFromBytes};
use sui_types::gas::SuiGasStatus;
use sui_types::in_memory_storage::InMemoryStorage;
use sui_types::messages::CallArg;
Expand Down Expand Up @@ -101,38 +99,6 @@ impl Genesis {
}))
}

#[allow(clippy::mutable_key_type)]
pub fn narwhal_worker_cache(&self) -> narwhal_config::SharedWorkerCache {
let workers = self
.validator_set
.iter()
.map(|validator| {
let name = AuthorityPublicKey::from_bytes(validator.protocol_key().as_ref())
.expect("Can't get protocol key");
let workers = [(
0, // worker_id
narwhal_config::WorkerInfo {
name: NetworkPublicKey::from_bytes(validator.worker_key().as_ref())
.expect("Can't get worker key"),
transactions: validator.narwhal_consensus_address.clone(),
worker_address: validator.narwhal_worker_address.clone(),
internal_worker_address: validator.narwhal_internal_worker_address.clone(),
},
)]
.into_iter()
.collect();
let worker_index = narwhal_config::WorkerIndex(workers);

(name, worker_index)
})
.collect();
narwhal_config::WorkerCache {
workers,
epoch: self.epoch() as narwhal_config::Epoch,
}
.into()
}

pub fn sui_system_object(&self) -> SuiSystemState {
let sui_system_object = self
.objects()
Expand Down Expand Up @@ -689,8 +655,6 @@ mod test {
p2p_address: utils::new_udp_network_address(),
narwhal_primary_address: utils::new_udp_network_address(),
narwhal_worker_address: utils::new_udp_network_address(),
narwhal_internal_worker_address: None,
narwhal_consensus_address: utils::new_tcp_network_address(),
};
let pop = generate_proof_of_possession(&key, account_key.public().into());
let builder = Builder::new()
Expand Down
18 changes: 10 additions & 8 deletions crates/sui-config/src/genesis_config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,17 @@ use crate::node::DEFAULT_GRPC_CONCURRENCY_LIMIT;
use crate::Config;
use crate::{utils, DEFAULT_COMMISSION_RATE, DEFAULT_GAS_PRICE, DEFAULT_STAKE};

// All information needed to build a NodeConfig for a validator.
#[derive(Serialize, Deserialize)]
pub struct ValidatorConfigInfo {
pub genesis_info: ValidatorGenesisInfo,
pub consensus_address: Multiaddr,
pub consensus_internal_worker_address: Option<Multiaddr>,
}

#[derive(Serialize, Deserialize)]
pub struct GenesisConfig {
pub validator_genesis_info: Option<Vec<ValidatorGenesisInfo>>,
pub validator_config_info: Option<Vec<ValidatorConfigInfo>>,
pub parameters: GenesisChainParameters,
pub committee_size: usize,
pub grpc_load_shed: Option<bool>,
Expand Down Expand Up @@ -108,8 +116,6 @@ pub struct ValidatorGenesisInfo {
pub commission_rate: u64,
pub narwhal_primary_address: Multiaddr,
pub narwhal_worker_address: Multiaddr,
pub narwhal_internal_worker_address: Option<Multiaddr>,
pub narwhal_consensus_address: Multiaddr,
}

impl ValidatorGenesisInfo {
Expand All @@ -131,8 +137,6 @@ impl ValidatorGenesisInfo {
commission_rate: DEFAULT_COMMISSION_RATE,
narwhal_primary_address: utils::new_udp_network_address(),
narwhal_worker_address: utils::new_udp_network_address(),
narwhal_internal_worker_address: None,
narwhal_consensus_address: utils::new_tcp_network_address(),
}
}

Expand Down Expand Up @@ -165,8 +169,6 @@ impl ValidatorGenesisInfo {
commission_rate: DEFAULT_COMMISSION_RATE,
narwhal_primary_address: make_udp_addr(2000 + port_offset),
narwhal_worker_address: make_udp_addr(3000 + port_offset),
narwhal_internal_worker_address: None,
narwhal_consensus_address: make_tcp_addr(4000 + port_offset),
}
}
}
Expand Down Expand Up @@ -295,7 +297,7 @@ impl GenesisConfig {
impl Default for GenesisConfig {
fn default() -> Self {
Self {
validator_genesis_info: None,
validator_config_info: None,
parameters: Default::default(),
committee_size: DEFAULT_NUMBER_OF_AUTHORITIES,
grpc_load_shed: None,
Expand Down
Loading

0 comments on commit d7d8bf8

Please sign in to comment.