From 647622c7db436eadc7ec29195c2ed62b7a4f13ea Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 17 Dec 2021 16:48:56 +1100 Subject: [PATCH 01/37] Implement proposer boost re-orging --- beacon_node/beacon_chain/src/beacon_chain.rs | 87 ++++++++++++++++++- beacon_node/beacon_chain/src/chain_config.rs | 5 ++ beacon_node/beacon_chain/src/errors.rs | 2 + beacon_node/src/cli.rs | 14 +++ beacon_node/src/config.rs | 12 +++ consensus/fork_choice/src/fork_choice.rs | 21 ++++- consensus/proto_array/src/error.rs | 1 + consensus/proto_array/src/lib.rs | 4 +- consensus/proto_array/src/proto_array.rs | 2 +- .../src/proto_array_fork_choice.rs | 84 +++++++++++++++++- scripts/local_testnet/beacon_node.sh | 4 +- 11 files changed, 229 insertions(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1f36e0e65ac..059b2c27bb3 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -34,7 +34,7 @@ use crate::observed_operations::{ObservationOutcome, ObservedOperations}; use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_BLOCK_ROOT}; use crate::persisted_fork_choice::PersistedForkChoice; use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache}; -use crate::snapshot_cache::SnapshotCache; +use crate::snapshot_cache::{BlockProductionPreState, SnapshotCache}; use crate::sync_committee_verification::{ Error as SyncCommitteeError, VerifiedSyncCommitteeMessage, VerifiedSyncContribution, }; @@ -101,6 +101,9 @@ pub const ATTESTATION_CACHE_LOCK_TIMEOUT: Duration = Duration::from_secs(1); /// validator pubkey cache. pub const VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT: Duration = Duration::from_secs(1); +/// The latest delay from the start of the slot at which to attempt a 1-slot re-org. +const MAX_RE_ORG_SLOT_DELAY: Duration = Duration::from_secs(2); + // These keys are all zero because they get stored in different columns, see `DBColumn` type. pub const BEACON_CHAIN_DB_KEY: Hash256 = Hash256::zero(); pub const OP_POOL_DB_KEY: Hash256 = Hash256::zero(); @@ -2723,8 +2726,18 @@ impl BeaconChain { .head_info() .map_err(BlockProductionError::UnableToGetHeadInfo)?; let (state, state_root_opt) = if head_info.slot < slot { + // Attempt an aggressive re-org if configured and the conditions are right. + if let Some(re_org_state) = self.get_state_for_re_org(slot, &head_info)? { + info!( + self.log, + "Proposing block to re-org current head"; + "slot" => slot, + "head" => %head_info.block_root, + ); + (re_org_state.pre_state, re_org_state.state_root) + } // Normal case: proposing a block atop the current head. Use the snapshot cache. - if let Some(pre_state) = self + else if let Some(pre_state) = self .snapshot_cache .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) .and_then(|snapshot_cache| { @@ -2769,6 +2782,76 @@ impl BeaconChain { ) } + fn get_state_for_re_org( + &self, + slot: Slot, + head_info: &HeadInfo, + ) -> Result>, BlockProductionError> { + if let Some(re_org_threshold) = self.config.re_org_threshold { + let canonical_head = head_info.block_root; + let slot_delay = self + .slot_clock + .seconds_from_current_slot_start(self.spec.seconds_per_slot) + .ok_or(BlockProductionError::UnableToReadSlot)?; + + // Check that we're producing a block one slot after the current head, and early enough + // in the slot to be able to propagate widely. + if head_info.slot + 1 == slot && slot_delay < MAX_RE_ORG_SLOT_DELAY { + // Is the current head weak and appropriate for re-orging? + let proposer_head = self.fork_choice.write().get_proposer_head( + slot, + canonical_head, + re_org_threshold, + )?; + if let Some(re_org_head) = proposer_head.re_org_head { + // Only attempt a re-org if we hit the snapshot cache. + if let Some(pre_state) = self + .snapshot_cache + .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) + .and_then(|snapshot_cache| { + snapshot_cache.get_state_for_block_production(re_org_head) + }) + { + debug!( + self.log, + "Attempting re-org due to weak head"; + "head" => ?canonical_head, + "re_org_head" => ?re_org_head, + "head_weight" => ?proposer_head.canonical_head_weight, + "re_org_weight" => ?proposer_head.re_org_weight_threshold, + ); + return Ok(Some(pre_state)); + } else { + debug!( + self.log, + "Not attempting re-org due to cache miss"; + "head" => ?canonical_head, + "re_org_head" => ?re_org_head, + "head_weight" => ?proposer_head.canonical_head_weight, + "re_org_weight" => ?proposer_head.re_org_weight_threshold, + ); + } + } else { + debug!( + self.log, + "Not attempting re-org due to strong head"; + "head" => ?canonical_head, + "head_weight" => ?proposer_head.canonical_head_weight, + "re_org_weight" => ?proposer_head.re_org_weight_threshold, + ); + } + } else { + debug!( + self.log, + "Not attempting re-org due to slot distance"; + "head" => ?canonical_head, + ); + } + } + + Ok(None) + } + /// Produce a block for some `slot` upon the given `state`. /// /// Typically the `self.produce_block()` function should be used, instead of calling this diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index 4aee06d468c..6800cc145da 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -1,6 +1,8 @@ use serde_derive::{Deserialize, Serialize}; use types::Checkpoint; +pub const DEFAULT_RE_ORG_THRESHOLD: u64 = 10; + #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] pub struct ChainConfig { /// Maximum number of slots to skip when importing a consensus message (e.g., block, @@ -18,6 +20,8 @@ pub struct ChainConfig { pub enable_lock_timeouts: bool, /// The max size of a message that can be sent over the network. pub max_network_size: usize, + /// Maximum percentage of weight at which to attempt re-orging the canonical head. + pub re_org_threshold: Option, } impl Default for ChainConfig { @@ -28,6 +32,7 @@ impl Default for ChainConfig { reconstruct_historic_states: false, enable_lock_timeouts: true, max_network_size: 10 * 1_048_576, // 10M + re_org_threshold: None, } } } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 6b9af787d70..e6edfa1e05a 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -169,6 +169,7 @@ pub enum BlockProductionError { UnableToProduceAtSlot(Slot), SlotProcessingError(SlotProcessingError), BlockProcessingError(BlockProcessingError), + ForkChoiceError(ForkChoiceError), Eth1ChainError(Eth1ChainError), BeaconStateError(BeaconStateError), StateAdvanceError(StateAdvanceError), @@ -194,3 +195,4 @@ easy_from_to!(BeaconStateError, BlockProductionError); easy_from_to!(SlotProcessingError, BlockProductionError); easy_from_to!(Eth1ChainError, BlockProductionError); easy_from_to!(StateAdvanceError, BlockProductionError); +easy_from_to!(ForkChoiceError, BlockProductionError); diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index afcb125c274..ff995db2c5c 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -625,4 +625,18 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { experimental as it may obscure performance issues.") .takes_value(false) ) + .arg( + Arg::with_name("enable-proposer-re-orgs") + .long("enable-proposer-re-orgs") + .help("Attempt to re-org out weak/late blocks from other proposers \ + (dangerous, experimental)") + .takes_value(true) + ) + .arg( + Arg::with_name("proposer-re-org-fraction") + .long("proposer-re-org-fraction") + .help("Percentage of vote weight below which to attempt a proposer re-org") + .requires("enable-proposer-re-orgs") + .takes_value(true) + ) } diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index e9e3e2cd5b7..9962e4f8a30 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1,3 +1,4 @@ +use beacon_chain::chain_config::DEFAULT_RE_ORG_THRESHOLD; use clap::ArgMatches; use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG; use client::{ClientConfig, ClientGenesis}; @@ -562,6 +563,17 @@ pub fn get_config( client_config.chain.enable_lock_timeouts = false; } + if let Some(enable_re_orgs) = clap_utils::parse_optional(cli_args, "enable-proposer-re-orgs")? { + if enable_re_orgs { + client_config.chain.re_org_threshold = Some( + clap_utils::parse_optional(cli_args, "proposer-re-org-fraction")? + .unwrap_or(DEFAULT_RE_ORG_THRESHOLD), + ); + } else { + client_config.chain.re_org_threshold = None; + } + } + Ok(client_config) } diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 86b32aab1a4..cc6dd177567 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -1,5 +1,5 @@ use crate::ForkChoiceStore; -use proto_array::{Block as ProtoBlock, ExecutionStatus, ProtoArrayForkChoice}; +use proto_array::{Block as ProtoBlock, ExecutionStatus, ProposerHead, ProtoArrayForkChoice}; use ssz_derive::{Decode, Encode}; use std::cmp::Ordering; use std::marker::PhantomData; @@ -412,6 +412,25 @@ where .map_err(Into::into) } + pub fn get_proposer_head( + &mut self, + current_slot: Slot, + canonical_head: Hash256, + re_org_threshold: u64, + ) -> Result> { + // Calling `update_time` is essential, as it needs to dequeue attestations from the previous + // slot so we can see how many attesters voted for the canonical head. + self.update_time(current_slot)?; + + self.proto_array + .get_proposer_head::( + self.fc_store.justified_balances(), + canonical_head, + re_org_threshold, + ) + .map_err(Into::into) + } + /// Returns `true` if the given `store` should be updated to set /// `state.current_justified_checkpoint` its `justified_checkpoint`. /// diff --git a/consensus/proto_array/src/error.rs b/consensus/proto_array/src/error.rs index adb10c035d6..74302f89b64 100644 --- a/consensus/proto_array/src/error.rs +++ b/consensus/proto_array/src/error.rs @@ -14,6 +14,7 @@ pub enum Error { InvalidNodeDelta(usize), DeltaOverflow(usize), ProposerBoostOverflow(usize), + UniqueWeightOverflow(Hash256), IndexOverflow(&'static str), InvalidDeltaLen { deltas: usize, diff --git a/consensus/proto_array/src/lib.rs b/consensus/proto_array/src/lib.rs index 216d189fb2a..3340676d838 100644 --- a/consensus/proto_array/src/lib.rs +++ b/consensus/proto_array/src/lib.rs @@ -4,7 +4,9 @@ mod proto_array; mod proto_array_fork_choice; mod ssz_container; -pub use crate::proto_array_fork_choice::{Block, ExecutionStatus, ProtoArrayForkChoice}; +pub use crate::proto_array_fork_choice::{ + Block, ExecutionStatus, ProposerHead, ProtoArrayForkChoice, +}; pub use error::Error; pub mod core { diff --git a/consensus/proto_array/src/proto_array.rs b/consensus/proto_array/src/proto_array.rs index 465ef9d4fc7..beb57aa412e 100644 --- a/consensus/proto_array/src/proto_array.rs +++ b/consensus/proto_array/src/proto_array.rs @@ -579,7 +579,7 @@ impl ProtoArray { /// Returns `None` if there is an overflow or underflow when calculating the score. /// /// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#get_latest_attesting_balance -fn calculate_proposer_boost( +pub fn calculate_proposer_boost( validator_balances: &[u64], proposer_score_boost: u64, ) -> Option { diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index 891eafabe9a..c41a4edda6f 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -1,5 +1,5 @@ use crate::error::Error; -use crate::proto_array::{ProposerBoost, ProtoArray}; +use crate::proto_array::{calculate_proposer_boost, ProposerBoost, ProtoArray}; use crate::ssz_container::SszContainer; use serde_derive::{Deserialize, Serialize}; use ssz::{Decode, Encode}; @@ -92,11 +92,26 @@ where &mut self.0[i] } + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + pub fn iter_mut(&mut self) -> impl Iterator { self.0.iter_mut() } } +/// Information about the proposer head used for opportunistic re-orgs. +#[derive(Default, Clone)] +pub struct ProposerHead { + /// If set, the head block that the proposer should build upon. + pub re_org_head: Option, + /// The weight difference between the canonical head and its parent. + pub canonical_head_weight: Option, + /// The computed fraction of the active committee balance below which we can re-org. + pub re_org_weight_threshold: Option, +} + #[derive(PartialEq)] pub struct ProtoArrayForkChoice { pub(crate) proto_array: ProtoArray, @@ -214,6 +229,73 @@ impl ProtoArrayForkChoice { .map_err(|e| format!("find_head failed: {:?}", e)) } + pub fn get_proposer_head( + &self, + justified_state_balances: &[u64], + canonical_head: Hash256, + re_org_vote_fraction: u64, + ) -> Result { + let nodes = self + .proto_array + .iter_nodes(&canonical_head) + .take(2) + .collect::>(); + if nodes.len() != 2 { + return Ok(ProposerHead::default()); + } + let head_node = nodes[0]; + let parent_node = nodes[1]; + + // Re-org conditions. + let is_single_slot_re_org = parent_node.slot + 1 == head_node.slot; + let re_org_weight_threshold = + calculate_proposer_boost::(justified_state_balances, re_org_vote_fraction) + .ok_or_else(|| { + "overflow calculating committee weight for proposer boost".to_string() + })?; + let canonical_head_weight = self + .get_block_unique_weight(canonical_head, justified_state_balances) + .map_err(|e| format!("overflow calculating head weight: {:?}", e))?; + let is_weak_head = canonical_head_weight < re_org_weight_threshold; + + let re_org_head = (is_single_slot_re_org && is_weak_head).then(|| parent_node.root); + + Ok(ProposerHead { + re_org_head, + canonical_head_weight: Some(canonical_head_weight), + re_org_weight_threshold: Some(re_org_weight_threshold), + }) + } + + /// Compute the sum of attester balances of attestations to a specific block root. + /// + /// This weight is the weight unique to the block, *not* including the weight of its ancestors. + /// + /// Any `proposer_boost` in effect is ignored: only attestations are counted. + fn get_block_unique_weight( + &self, + block_root: Hash256, + justified_balances: &[u64], + ) -> Result { + let mut unique_weight = 0u64; + for (validator_index, vote) in self.votes.iter().enumerate() { + // Check the `next_root` as we care about the most recent attestations, including ones + // from the previous slot that have just been dequeued but haven't run fully through + // fork choice yet. + if vote.next_root == block_root { + let validator_balance = justified_balances + .get(validator_index) + .copied() + .unwrap_or(0); + + unique_weight = unique_weight + .checked_add(validator_balance) + .ok_or(Error::UniqueWeightOverflow(block_root))?; + } + } + Ok(unique_weight) + } + pub fn maybe_prune(&mut self, finalized_root: Hash256) -> Result<(), String> { self.proto_array .maybe_prune(finalized_root) diff --git a/scripts/local_testnet/beacon_node.sh b/scripts/local_testnet/beacon_node.sh index 883c6660294..caab7868856 100755 --- a/scripts/local_testnet/beacon_node.sh +++ b/scripts/local_testnet/beacon_node.sh @@ -51,4 +51,6 @@ exec lighthouse \ --port $network_port \ --http-port $http_port \ --disable-packet-filter \ - --target-peers $((BN_COUNT - 1)) + --target-peers $((BN_COUNT - 1)) \ + --enable-proposer-re-orgs true \ + --proposer-re-org-fraction 60 From 7e52eb36caddcf08de9d6f6b2e56f61c1e591574 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 4 Mar 2022 13:20:11 +1100 Subject: [PATCH 02/37] Add --block-delay-ms and tweak local testnets --- beacon_node/beacon_chain/src/beacon_chain.rs | 9 ++++++++ .../beacon_chain/src/snapshot_cache.rs | 7 +++++-- lcli/src/main.rs | 8 +++++++ lcli/src/new_testnet.rs | 4 ++++ scripts/local_testnet/beacon_node.sh | 4 ++-- scripts/local_testnet/setup.sh | 1 + scripts/local_testnet/vars.env | 9 ++++++++ validator_client/src/block_service.rs | 21 +++++++++++++++++++ validator_client/src/cli.rs | 12 +++++++++++ validator_client/src/config.rs | 13 ++++++++++++ validator_client/src/lib.rs | 1 + 11 files changed, 85 insertions(+), 4 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index af2cd8c0f41..5805308b422 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2957,6 +2957,15 @@ impl BeaconChain { head_info: &HeadInfo, ) -> Result>, BlockProductionError> { if let Some(re_org_threshold) = self.config.re_org_threshold { + if self.spec.proposer_score_boost.is_none() { + warn!( + self.log, + "Ignoring proposer re-org configuration"; + "reason" => "this network does not have proposer boosting enabled" + ); + return Ok(None); + } + let canonical_head = head_info.block_root; let slot_delay = self .slot_clock diff --git a/beacon_node/beacon_chain/src/snapshot_cache.rs b/beacon_node/beacon_chain/src/snapshot_cache.rs index f4bbae8a32e..5695bcecfbf 100644 --- a/beacon_node/beacon_chain/src/snapshot_cache.rs +++ b/beacon_node/beacon_chain/src/snapshot_cache.rs @@ -12,7 +12,10 @@ pub const DEFAULT_SNAPSHOT_CACHE_SIZE: usize = 4; /// The minimum block delay to clone the state in the cache instead of removing it. /// This helps keep block processing fast during re-orgs from late blocks. -const MINIMUM_BLOCK_DELAY_FOR_CLONE: Duration = Duration::from_secs(6); +fn minimum_block_delay_for_clone(seconds_per_slot: u64) -> Duration { + // If the block arrived at the attestation deadline or later, it might get re-orged. + Duration::from_secs(seconds_per_slot) / 3 +} /// This snapshot is to be used for verifying a child of `self.beacon_block`. #[derive(Debug)] @@ -257,7 +260,7 @@ impl SnapshotCache { return (cache.clone_as_pre_state(), true); } if let Some(delay) = block_delay { - if delay >= MINIMUM_BLOCK_DELAY_FOR_CLONE + if delay >= minimum_block_delay_for_clone(spec.seconds_per_slot) && delay <= Duration::from_secs(spec.seconds_per_slot) * 4 { return (cache.clone_as_pre_state(), true); diff --git a/lcli/src/main.rs b/lcli/src/main.rs index 9af4b255488..bba3af3145e 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -501,6 +501,14 @@ fn main() { .takes_value(true) .help("The genesis time when generating a genesis state."), ) + .arg( + Arg::with_name("proposer-score-boost") + .long("proposer-score-boost") + .value_name("INTEGER") + .takes_value(true) + .help("The proposer score boost to apply as a percentage, e.g. 70 = 70%"), + ) + ) .subcommand( SubCommand::with_name("check-deposit-data") diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index 5254ff5a62e..b2760829cb8 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -58,6 +58,10 @@ pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul spec.genesis_fork_version = v; } + if let Some(proposer_score_boost) = parse_optional(matches, "proposer-score-boost")? { + spec.proposer_score_boost = Some(proposer_score_boost); + } + if let Some(fork_epoch) = parse_optional(matches, "altair-fork-epoch")? { spec.altair_fork_epoch = Some(fork_epoch); } diff --git a/scripts/local_testnet/beacon_node.sh b/scripts/local_testnet/beacon_node.sh index f945e30f984..55e8874fa72 100755 --- a/scripts/local_testnet/beacon_node.sh +++ b/scripts/local_testnet/beacon_node.sh @@ -55,5 +55,5 @@ exec lighthouse \ --http-port $http_port \ --disable-packet-filter \ --target-peers $((BN_COUNT - 1)) \ - --enable-proposer-re-orgs true \ - --proposer-re-org-fraction 60 + --enable-proposer-re-orgs "$ENABLE_PROPOSER_RE_ORGS" \ + --proposer-re-org-fraction "$PROPOSER_RE_ORG_FRACTION" diff --git a/scripts/local_testnet/setup.sh b/scripts/local_testnet/setup.sh index 6f0b070915a..49d8edd119e 100755 --- a/scripts/local_testnet/setup.sh +++ b/scripts/local_testnet/setup.sh @@ -36,6 +36,7 @@ lcli \ --eth1-follow-distance 1 \ --seconds-per-slot $SECONDS_PER_SLOT \ --seconds-per-eth1-block $SECONDS_PER_ETH1_BLOCK \ + --proposer-score-boost "$PROPOSER_SCORE_BOOST" \ --force echo Specification generated at $TESTNET_DIR. diff --git a/scripts/local_testnet/vars.env b/scripts/local_testnet/vars.env index 208fbb6d856..8fe2e4006cf 100644 --- a/scripts/local_testnet/vars.env +++ b/scripts/local_testnet/vars.env @@ -44,5 +44,14 @@ SECONDS_PER_SLOT=3 # Seconds per Eth1 block SECONDS_PER_ETH1_BLOCK=1 +# Proposer score boost percentage +PROPOSER_SCORE_BOOST=70 + +# Enable re-orgs of late blocks using the proposer boost? +ENABLE_PROPOSER_RE_ORGS=true + +# Minimum percentage of the vote that a block must have in order to not get re-orged. +PROPOSER_RE_ORG_FRACTION=50 + # Command line arguments for validator client VC_ARGS="" diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 0cba70481f1..831645e3ac8 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -9,7 +9,9 @@ use slog::{crit, debug, error, info, trace, warn}; use slot_clock::SlotClock; use std::ops::Deref; use std::sync::Arc; +use std::time::Duration; use tokio::sync::mpsc; +use tokio::time::sleep; use types::{EthSpec, PublicKeyBytes, Slot}; /// Builds a `BlockService`. @@ -20,6 +22,7 @@ pub struct BlockServiceBuilder { context: Option>, graffiti: Option, graffiti_file: Option, + block_delay: Option, } impl BlockServiceBuilder { @@ -31,6 +34,7 @@ impl BlockServiceBuilder { context: None, graffiti: None, graffiti_file: None, + block_delay: None, } } @@ -64,6 +68,11 @@ impl BlockServiceBuilder { self } + pub fn block_delay(mut self, block_delay: Option) -> Self { + self.block_delay = block_delay; + self + } + pub fn build(self) -> Result, String> { Ok(BlockService { inner: Arc::new(Inner { @@ -81,6 +90,7 @@ impl BlockServiceBuilder { .ok_or("Cannot build BlockService without runtime_context")?, graffiti: self.graffiti, graffiti_file: self.graffiti_file, + block_delay: self.block_delay, }), }) } @@ -94,6 +104,7 @@ pub struct Inner { context: RuntimeContext, graffiti: Option, graffiti_file: Option, + block_delay: Option, } /// Attempts to produce attestations for any block producer(s) at the start of the epoch. @@ -138,6 +149,16 @@ impl BlockService { async move { while let Some(notif) = notification_rx.recv().await { let service = self.clone(); + + if let Some(delay) = service.block_delay { + debug!( + service.context.log(), + "Delaying block production by {}ms", + delay.as_millis() + ); + sleep(delay).await; + } + service.do_update(notif).await.ok(); } debug!(log, "Block service shutting down"); diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 49a8f581677..1321a4f1291 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -258,4 +258,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { immediately.") .takes_value(false), ) + /* + * Experimental/development options. + */ + .arg( + Arg::with_name("block-delay-ms") + .long("block-delay-ms") + .value_name("MILLIS") + .hidden(true) + .help("Time to delay block production from the start of the slot. Should only be \ + used for testing.") + .takes_value(true), + ) } diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index cb9f80eab59..6b5d2a031c1 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -14,6 +14,7 @@ use slog::{info, warn, Logger}; use std::fs; use std::net::Ipv4Addr; use std::path::PathBuf; +use std::time::Duration; use types::{Address, GRAFFITI_BYTES_LEN}; pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/"; @@ -58,6 +59,10 @@ pub struct Config { /// A list of custom certificates that the validator client will additionally use when /// connecting to a beacon node over SSL/TLS. pub beacon_nodes_tls_certs: Option>, + /// Delay from the start of the slot to wait before publishing a block. + /// + /// This is *not* recommended in prod and should only be used for testing. + pub block_delay: Option, } impl Default for Config { @@ -91,6 +96,7 @@ impl Default for Config { monitoring_api: None, enable_doppelganger_protection: false, beacon_nodes_tls_certs: None, + block_delay: None, } } } @@ -306,6 +312,13 @@ impl Config { config.enable_doppelganger_protection = true; } + /* + * Experimental + */ + if let Some(delay_ms) = parse_optional::(cli_args, "block-delay-ms")? { + config.block_delay = Some(Duration::from_millis(delay_ms)); + } + Ok(config) } } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index c58ac25f1f9..a9deec44f27 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -400,6 +400,7 @@ impl ProductionValidatorClient { .runtime_context(context.service_context("block".into())) .graffiti(config.graffiti) .graffiti_file(config.graffiti_file.clone()) + .block_delay(config.block_delay) .build()?; let attestation_service = AttestationServiceBuilder::new() From 1c8cd67c1c33d59d7d6fc21a2b14e63d4a3fdb79 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 4 Mar 2022 15:38:52 +1100 Subject: [PATCH 03/37] Make max delay for re-org dynamic This will help chains with non-standard slot times, e.g. Gnosis Chain. --- beacon_node/beacon_chain/src/beacon_chain.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5805308b422..b43b8279516 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -104,7 +104,10 @@ pub const ATTESTATION_CACHE_LOCK_TIMEOUT: Duration = Duration::from_secs(1); pub const VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT: Duration = Duration::from_secs(1); /// The latest delay from the start of the slot at which to attempt a 1-slot re-org. -const MAX_RE_ORG_SLOT_DELAY: Duration = Duration::from_secs(2); +fn max_re_org_slot_delay(seconds_per_slot: u64) -> Duration { + // Allow at least half of the attestation deadline for the block to propagate. + Duration::from_secs(seconds_per_slot) / 6 +} // These keys are all zero because they get stored in different columns, see `DBColumn` type. pub const BEACON_CHAIN_DB_KEY: Hash256 = Hash256::zero(); @@ -2974,7 +2977,9 @@ impl BeaconChain { // Check that we're producing a block one slot after the current head, and early enough // in the slot to be able to propagate widely. - if head_info.slot + 1 == slot && slot_delay < MAX_RE_ORG_SLOT_DELAY { + if head_info.slot + 1 == slot + && slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot) + { // Is the current head weak and appropriate for re-orging? let proposer_head = self.fork_choice.write().get_proposer_head( slot, From e43d6be61943720ea7df64467e3746bc32d52b49 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 4 Mar 2022 16:40:05 +1100 Subject: [PATCH 04/37] CLI flag tests --- lighthouse/tests/beacon_node.rs | 38 ++++++++++++++++++++++++++++ lighthouse/tests/validator_client.rs | 18 +++++++++++++ 2 files changed, 56 insertions(+) diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 7de201bc3f1..a33f1076c46 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1,6 +1,7 @@ use beacon_node::ClientConfig as Config; use crate::exec::{CommandLineTestExec, CompletedTest}; +use beacon_node::beacon_chain::chain_config::DEFAULT_RE_ORG_THRESHOLD; use lighthouse_network::PeerId; use std::fs::File; use std::io::Write; @@ -956,3 +957,40 @@ fn ensure_panic_on_failed_launch() { assert_eq!(slasher_config.chunk_size, 10); }); } + +#[test] +fn enable_proposer_re_orgs_true() { + CommandLineTest::new() + .flag("enable-proposer-re-orgs", Some("true")) + .run() + .with_config(|config| { + assert_eq!( + config.chain.re_org_threshold, + Some(DEFAULT_RE_ORG_THRESHOLD) + ) + }); +} + +#[test] +fn enable_proposer_re_orgs_false() { + CommandLineTest::new() + .flag("enable-proposer-re-orgs", Some("false")) + .run() + .with_config(|config| assert_eq!(config.chain.re_org_threshold, None)); +} + +#[test] +fn enable_proposer_re_orgs_default() { + CommandLineTest::new() + .run() + .with_config(|config| assert_eq!(config.chain.re_org_threshold, None)); +} + +#[test] +fn proposer_re_org_fraction() { + CommandLineTest::new() + .flag("enable-proposer-re-orgs", Some("true")) + .flag("proposer-re-org-fraction", Some("90")) + .run() + .with_config(|config| assert_eq!(config.chain.re_org_threshold, Some(90))); +} diff --git a/lighthouse/tests/validator_client.rs b/lighthouse/tests/validator_client.rs index 76315daaa9a..8d6b6c710a7 100644 --- a/lighthouse/tests/validator_client.rs +++ b/lighthouse/tests/validator_client.rs @@ -426,3 +426,21 @@ fn no_doppelganger_protection_flag() { .run() .with_config(|config| assert!(!config.enable_doppelganger_protection)); } +#[test] +fn block_delay_ms() { + CommandLineTest::new() + .flag("block-delay-ms", Some("2000")) + .run() + .with_config(|config| { + assert_eq!( + config.block_delay, + Some(std::time::Duration::from_millis(2000)) + ) + }); +} +#[test] +fn no_block_delay_ms() { + CommandLineTest::new() + .run() + .with_config(|config| assert_eq!(config.block_delay, None)); +} From a9622958ef20852a51d7f16b71e870b53304fb15 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 5 May 2022 09:24:34 +1000 Subject: [PATCH 05/37] Don't re-org on the first slot of the epoch --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index da7404128ee..7cbfcefacd4 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3017,8 +3017,10 @@ impl BeaconChain { .ok_or(BlockProductionError::UnableToReadSlot)?; // Check that we're producing a block one slot after the current head, and early enough - // in the slot to be able to propagate widely. + // in the slot to be able to propagate widely. For simplicity of analysis, also avoid + // proposing a re-org block in the first slot of the epoch, as if head_info.slot + 1 == slot + && slot % T::EthSpec::slots_per_epoch() != 0 && slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot) { // Is the current head weak and appropriate for re-orging? From 7e6a1884ec44d8bd897425d761da8fb0dd6c68cc Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 5 May 2022 09:24:52 +1000 Subject: [PATCH 06/37] Start writing tests (WIP) --- beacon_node/http_api/tests/common.rs | 21 ++++++++++++++++--- .../http_api/tests/interactive_tests.rs | 5 +++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/beacon_node/http_api/tests/common.rs b/beacon_node/http_api/tests/common.rs index 06466c43bb4..79e427f1aad 100644 --- a/beacon_node/http_api/tests/common.rs +++ b/beacon_node/http_api/tests/common.rs @@ -46,13 +46,28 @@ pub struct ApiServer> { pub external_peer_id: PeerId, } +type Mutator = BoxedMutator, MemoryStore>; + impl InteractiveTester { pub async fn new(spec: Option, validator_count: usize) -> Self { - let harness = BeaconChainHarness::builder(E::default()) + Self::new_with_mutator(spec, validator_count, None) + } + + pub async fn new_with_mutator( + spec: Option, + validator_count: usize, + mutator: Option>, + ) -> Self { + let harness_builder = BeaconChainHarness::builder(E::default()) .spec_or_default(spec) .deterministic_keypairs(validator_count) - .fresh_ephemeral_store() - .build(); + .fresh_ephemeral_store(); + + if let Some(mutator) = mutator { + harness_builder = harness_builder.initial_mutator(mutator); + } + + let harness = harness_builder.build(); let ApiServer { server, diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 64ce3b6566c..018062a965e 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -30,3 +30,8 @@ async fn deposit_contract_custom_network() { assert_eq!(result, expected); } + +// Test that the beacon node will try to perform proposer boost re-orgs on late blocks when +// configured. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub fn proposer_boost_re_org_success() {} From 142c0c57664d1b149d199cddfaa69d446023862a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 15 Aug 2022 14:29:02 +1000 Subject: [PATCH 07/37] Fix merge snafu --- lighthouse/tests/validator_client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/lighthouse/tests/validator_client.rs b/lighthouse/tests/validator_client.rs index 44d30b3c040..75cabe0e91a 100644 --- a/lighthouse/tests/validator_client.rs +++ b/lighthouse/tests/validator_client.rs @@ -460,5 +460,4 @@ fn no_strict_fee_recipient_flag() { CommandLineTest::new() .run() .with_config(|config| assert!(!config.strict_fee_recipient)); ->>>>>>> origin/unstable } From dd227fce9e5965334af067dc96d5c9c834c3a372 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 16 Aug 2022 14:47:53 +1000 Subject: [PATCH 08/37] Suppress fork choice updates for EL support! --- beacon_node/beacon_chain/src/beacon_chain.rs | 116 ++++++++++++++++++- beacon_node/execution_layer/src/lib.rs | 3 +- 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 2f480f0862e..1cc092b3aad 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3223,6 +3223,9 @@ impl BeaconChain { Ok((state, state_root_opt)) } + /// Fetch the beacon state to use for producing a block if a 1-slot proposer re-org is viable. + /// + /// This function will return `Ok(None)` if proposer re-orgs are disabled. fn get_state_for_re_org( &self, slot: Slot, @@ -3234,7 +3237,7 @@ impl BeaconChain { warn!( self.log, "Ignoring proposer re-org configuration"; - "reason" => "this network does not have proposer boosting enabled" + "reason" => "this network does not have proposer boost enabled" ); return Ok(None); } @@ -3246,10 +3249,16 @@ impl BeaconChain { // Check that we're producing a block one slot after the current head, and early enough // in the slot to be able to propagate widely. For simplicity of analysis, also avoid - // proposing a re-org block in the first slot of the epoch, as + // proposing a re-org block in the first slot of the epoch, as re-orging the last block + // of the previous epoch could change the proposer shuffling. + // + // Additionally, do a quick check that the current canonical head block was observed + // late. This aligns with `should_suppress_fork_choice_update` and prevents + // unnecessarily taking the fork choice write lock. if head_slot + 1 == slot && slot % T::EthSpec::slots_per_epoch() != 0 && slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot) + && self.block_observed_after_attestation_deadline(canonical_head, head_slot) { // Is the current head weak and appropriate for re-orging? let proposer_head = self @@ -3305,6 +3314,93 @@ impl BeaconChain { Ok(None) } + /// Determine whether a fork choice update to the execution layer should be suppressed. + /// + /// This is *only* necessary when proposer re-orgs are enabled, because we have to prevent the + /// execution layer from enshrining the block we want to re-org as the head. + /// + /// This function uses heuristics that align quite closely but not exactly with the re-org + /// conditions set out in `get_state_for_re_org` and `get_proposer_head`. The differences are + /// documented below. + async fn should_suppress_fork_choice_update( + &self, + current_slot: Slot, + head_block_root: Hash256, + ) -> Result { + // Never supress if proposer re-orgs are disabled. + if self.config.re_org_threshold.is_none() { + return Ok(false); + } + + // Load details of the head block and its parent from fork choice. + let (head_slot, parent_root, parent_slot) = { + let nodes = self + .canonical_head + .fork_choice_read_lock() + .proto_array() + .core_proto_array() + .iter_nodes(&head_block_root) + .take(2) + .cloned() + .collect::>(); + + if nodes.len() != 2 { + return Ok(false); + } + + (nodes[0].slot, nodes[1].root, nodes[1].slot) + }; + + // The slot of our potential re-org block is always 1 greater than the head block because we + // only attempt single-slot re-orgs. + let re_org_block_slot = head_slot + 1; + + // Suppress only during the head block's slot, or at most until the end of the next slot. + // If our proposal fails entirely we will attest to the wrong head during + // `re_org_block_slot` and only re-align with the canonical chain 500ms before the start of + // the next slot (i.e. `head_slot + 2`). + let current_slot_ok = head_slot == current_slot || re_org_block_slot == current_slot; + + // Only attempt single slot re-orgs, and not at epoch boundaries. + let block_slot_ok = + parent_slot + 1 == head_slot && re_org_block_slot % T::EthSpec::slots_per_epoch() != 0; + + // Check that this node has a proposer prepared to execute a re-org. + let prepared_for_re_org = self + .execution_layer + .as_ref() + .ok_or(Error::ExecutionLayerMissing)? + .payload_attributes(re_org_block_slot, parent_root) + .await + .is_some(); + + // Check that the head block arrived late and is vulnerable to a re-org. This check is only + // a heuristic compared to the proper weight check in `get_state_for_re_org`, the reason + // being that we may have only *just* received the block and not yet processed any + // attestations for it. We also can't dequeue attestations for the block during the + // current slot, which would be necessary for determining its weight. + let head_block_late = + self.block_observed_after_attestation_deadline(head_block_root, head_slot); + + let might_re_org = + current_slot_ok && block_slot_ok && prepared_for_re_org && head_block_late; + + Ok(might_re_org) + } + + /// Check if the block with `block_root` was observed after the attestation deadline of `slot`. + fn block_observed_after_attestation_deadline(&self, block_root: Hash256, slot: Slot) -> bool { + let block_delays = self.block_times_cache.read().get_block_delays( + block_root, + self.slot_clock + .start_of(slot) + .unwrap_or_else(|| Duration::from_secs(0)), + ); + block_delays.observed.map_or(false, |delay| { + delay > self.slot_clock.unagg_attestation_production_delay() + }) + } + /// Produce a block for some `slot` upon the given `state`. /// /// Typically the `self.produce_block()` function should be used, instead of calling this @@ -3974,6 +4070,7 @@ impl BeaconChain { "already_known" => already_known, "prepare_slot" => prepare_slot, "validator" => proposer, + "parent_root" => ?head_root, ); } @@ -4120,6 +4217,21 @@ impl BeaconChain { } }; + // Determine whether to suppress the forkchoiceUpdated message if we want to re-org + // the current head at the next slot. + if self + .should_suppress_fork_choice_update(current_slot, head_block_root) + .await? + { + debug!( + self.log, + "Suppressing fork choice update"; + "head_block_root" => ?head_block_root, + "current_slot" => current_slot, + ); + return Ok(()); + } + let forkchoice_updated_response = execution_layer .notify_forkchoice_updated( head_hash, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index e7bbc6cd5eb..8c43316748b 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -982,12 +982,13 @@ impl ExecutionLayer { &[metrics::FORKCHOICE_UPDATED], ); - trace!( + debug!( self.log(), "Issuing engine_forkchoiceUpdated"; "finalized_block_hash" => ?finalized_block_hash, "justified_block_hash" => ?justified_block_hash, "head_block_hash" => ?head_block_hash, + "current_slot" => current_slot, ); let next_slot = current_slot + 1; From 1b51a7ba60975feff1fc839eb94e179a2279860c Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 17 Aug 2022 16:50:06 +1000 Subject: [PATCH 09/37] Basic proposer re-org test --- beacon_node/beacon_chain/src/beacon_chain.rs | 23 ++++-- beacon_node/beacon_chain/src/builder.rs | 6 ++ beacon_node/beacon_chain/src/test_utils.rs | 21 +++++ beacon_node/http_api/tests/common.rs | 9 ++- .../http_api/tests/interactive_tests.rs | 80 ++++++++++++++++++- common/task_executor/src/test_utils.rs | 7 ++ 6 files changed, 136 insertions(+), 10 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1cc092b3aad..af670ab2bbf 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3255,11 +3255,12 @@ impl BeaconChain { // Additionally, do a quick check that the current canonical head block was observed // late. This aligns with `should_suppress_fork_choice_update` and prevents // unnecessarily taking the fork choice write lock. - if head_slot + 1 == slot - && slot % T::EthSpec::slots_per_epoch() != 0 - && slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot) - && self.block_observed_after_attestation_deadline(canonical_head, head_slot) - { + let single_slot_re_org = + head_slot + 1 == slot && slot % T::EthSpec::slots_per_epoch() != 0; + let proposing_on_time = slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot); + let head_late = + self.block_observed_after_attestation_deadline(canonical_head, head_slot); + if single_slot_re_org && proposing_on_time && head_late { // Is the current head weak and appropriate for re-orging? let proposer_head = self .canonical_head @@ -3302,6 +3303,18 @@ impl BeaconChain { "re_org_weight" => ?proposer_head.re_org_weight_threshold, ); } + } else if !head_late { + debug!( + self.log, + "Not attempting re-org of timely block"; + "head" => ?canonical_head, + ); + } else if !proposing_on_time { + debug!( + self.log, + "Not attempting re-org due to insufficient time"; + "head" => ?canonical_head, + ) } else { debug!( self.log, diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 252b7cef5a8..ae976e1b66e 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -158,6 +158,12 @@ where self } + /// Sets the proposer re-org threshold. + pub fn proposer_re_org_threshold(mut self, threshold: Option) -> Self { + self.chain_config.re_org_threshold = threshold; + self + } + /// Sets the store (database). /// /// Should generally be called early in the build chain. diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 9b625907032..908eaf3237c 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -319,6 +319,12 @@ where self } + pub fn logger(mut self, log: Logger) -> Self { + self.log = log.clone(); + self.runtime.set_logger(log); + self + } + /// This mutator will be run before the `store_mutator`. pub fn initial_mutator(mut self, mutator: BoxedMutator) -> Self { assert!( @@ -778,6 +784,21 @@ where sk.sign(message) } + /// Sign a beacon block using the proposer's key. + pub fn sign_beacon_block( + &self, + block: BeaconBlock, + state: &BeaconState, + ) -> SignedBeaconBlock { + let proposer_index = block.proposer_index() as usize; + block.sign( + &self.validator_keypairs[proposer_index].sk, + &state.fork(), + state.genesis_validators_root(), + &self.spec, + ) + } + /// Produces an "unaggregated" attestation for the given `slot` and `index` that attests to /// `beacon_block_root`. The provided `state` should match the `block.state_root` for the /// `block` identified by `beacon_block_root`. diff --git a/beacon_node/http_api/tests/common.rs b/beacon_node/http_api/tests/common.rs index 9ebbef4bce0..f20a468bd1f 100644 --- a/beacon_node/http_api/tests/common.rs +++ b/beacon_node/http_api/tests/common.rs @@ -1,5 +1,5 @@ use beacon_chain::{ - test_utils::{BeaconChainHarness, EphemeralHarnessType}, + test_utils::{BeaconChainHarness, BoxedMutator, EphemeralHarnessType}, BeaconChain, BeaconChainTypes, }; use eth2::{BeaconNodeHttpClient, Timeouts}; @@ -11,6 +11,7 @@ use lighthouse_network::{ types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield, SyncState}, ConnectedPoint, Enr, NetworkGlobals, PeerId, PeerManager, }; +use logging::test_logger; use network::NetworkMessage; use sensitive_url::SensitiveUrl; use slog::Logger; @@ -18,6 +19,7 @@ use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::Arc; use std::time::Duration; +use store::MemoryStore; use tokio::sync::{mpsc, oneshot}; use types::{ChainSpec, EthSpec}; @@ -50,7 +52,7 @@ type Mutator = BoxedMutator, MemoryStore>; impl InteractiveTester { pub async fn new(spec: Option, validator_count: usize) -> Self { - Self::new_with_mutator(spec, validator_count, None) + Self::new_with_mutator(spec, validator_count, None).await } pub async fn new_with_mutator( @@ -58,9 +60,10 @@ impl InteractiveTester { validator_count: usize, mutator: Option>, ) -> Self { - let harness_builder = BeaconChainHarness::builder(E::default()) + let mut harness_builder = BeaconChainHarness::builder(E::default()) .spec_or_default(spec) .deterministic_keypairs(validator_count) + .logger(test_logger()) .fresh_ephemeral_store(); if let Some(mutator) = mutator { diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 315b0d7d314..27ba5a8e7f6 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -2,6 +2,7 @@ use crate::common::*; use beacon_chain::test_utils::{AttestationStrategy, BlockStrategy}; use eth2::types::DepositContractData; +use slot_clock::SlotClock; use tree_hash::TreeHash; use types::{EthSpec, FullPayload, MainnetEthSpec, Slot}; @@ -36,8 +37,83 @@ async fn deposit_contract_custom_network() { // Test that the beacon node will try to perform proposer boost re-orgs on late blocks when // configured. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -pub fn proposer_boost_re_org_success() { - // FIXME(sproul): todo +pub async fn proposer_boost_re_org_success() { + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing + // `validator_count // 32`. + let validator_count = 32; + let num_initial: u64 = 31; + + let tester = InteractiveTester::::new_with_mutator( + None, + validator_count, + Some(Box::new(|builder| { + builder.proposer_re_org_threshold(Some(10)) + })), + ) + .await; + let harness = &tester.harness; + let slot_clock = &harness.chain.slot_clock; + + // Create some chain depth. + harness.advance_slot(); + harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + // We set up the following block graph, where B is a block that arrives late and is re-orged + // by C. + // + // A | B | - | + // ^ | - | C | + let slot_a = Slot::new(num_initial); + let slot_b = slot_a + 1; + let slot_c = slot_a + 2; + + let block_a_root = harness.head_block_root(); + let state_a = harness.get_current_state(); + + // Produce block B and process it halfway through the slot. + let (block_b, state_b) = harness.make_block(state_a.clone(), slot_b).await; + + let obs_time = slot_clock.start_of(slot_b).unwrap() + slot_clock.slot_duration() / 2; + slot_clock.set_current_time(obs_time); + harness.chain.block_times_cache.write().set_time_observed( + block_b.canonical_root(), + slot_b, + obs_time, + None, + None, + ); + harness.process_block_result(block_b).await.unwrap(); + + // Produce block C. + harness.advance_slot(); + harness.chain.per_slot_task().await; + + let proposer_index = state_b + .get_beacon_proposer_index(slot_c, &harness.chain.spec) + .unwrap(); + let randao_reveal = harness + .sign_randao_reveal(&state_b, proposer_index, slot_c) + .into(); + let unsigned_block_c = tester + .client + .get_validator_blocks(slot_c, &randao_reveal, None) + .await + .unwrap() + .data; + let block_c = harness.sign_beacon_block(unsigned_block_c, &state_b); + + // Block C should build on A. + assert_eq!(block_c.parent_root(), block_a_root); + + // Applying block C should cause a re-org from B to C. + let block_root_c = harness.process_block_result(block_c).await.unwrap().into(); + assert_eq!(harness.head_block_root(), block_root_c); } // Test that running fork choice before proposing results in selection of the correct head. diff --git a/common/task_executor/src/test_utils.rs b/common/task_executor/src/test_utils.rs index 7d59cdf022c..c6e5ad01e68 100644 --- a/common/task_executor/src/test_utils.rs +++ b/common/task_executor/src/test_utils.rs @@ -60,6 +60,13 @@ impl Drop for TestRuntime { } } +impl TestRuntime { + pub fn set_logger(&mut self, log: Logger) { + self.log = log.clone(); + self.task_executor.log = log; + } +} + pub fn null_logger() -> Result { let log_builder = NullLoggerBuilder; log_builder From caecc9a3fb8d5b02d01073f20cc4e470cafa7647 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 20 Sep 2022 18:39:05 +1000 Subject: [PATCH 10/37] Refine re-org conditions and extend tests --- beacon_node/beacon_chain/src/beacon_chain.rs | 103 ++++++++---------- beacon_node/beacon_chain/src/errors.rs | 1 + .../http_api/tests/interactive_tests.rs | 68 ++++++++++-- consensus/fork_choice/src/fork_choice.rs | 1 + .../src/proto_array_fork_choice.rs | 35 +++++- 5 files changed, 137 insertions(+), 71 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c822fe98dc4..f6322469648 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3263,25 +3263,24 @@ impl BeaconChain { .seconds_from_current_slot_start(self.spec.seconds_per_slot) .ok_or(BlockProductionError::UnableToReadSlot)?; - // Check that we're producing a block one slot after the current head, and early enough - // in the slot to be able to propagate widely. For simplicity of analysis, also avoid - // proposing a re-org block in the first slot of the epoch, as re-orging the last block - // of the previous epoch could change the proposer shuffling. + // Attempt a proposer re-org if: // - // Additionally, do a quick check that the current canonical head block was observed - // late. This aligns with `should_suppress_fork_choice_update` and prevents - // unnecessarily taking the fork choice write lock. - let single_slot_re_org = - head_slot + 1 == slot && slot % T::EthSpec::slots_per_epoch() != 0; + // 1. It seems we have time to propagate and still receive the proposer boost. + // 2. The current head block was seen late. + // 3. The `get_proposer_head` conditions from fork choice pass. let proposing_on_time = slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot); let head_late = self.block_observed_after_attestation_deadline(canonical_head, head_slot); - if single_slot_re_org && proposing_on_time && head_late { + let mut proposer_head = Default::default(); + let mut cache_hit = true; + + if proposing_on_time && head_late { // Is the current head weak and appropriate for re-orging? - let proposer_head = self + proposer_head = self .canonical_head .fork_choice_write_lock() .get_proposer_head(slot, canonical_head, re_org_threshold, &self.spec)?; + if let Some(re_org_head) = proposer_head.re_org_head { // Only attempt a re-org if we hit the snapshot cache. if let Some(pre_state) = self @@ -3300,44 +3299,23 @@ impl BeaconChain { "re_org_weight" => ?proposer_head.re_org_weight_threshold, ); return Ok(Some(pre_state)); - } else { - debug!( - self.log, - "Not attempting re-org due to cache miss"; - "head" => ?canonical_head, - "re_org_head" => ?re_org_head, - "head_weight" => ?proposer_head.canonical_head_weight, - "re_org_weight" => ?proposer_head.re_org_weight_threshold, - ); } - } else { - debug!( - self.log, - "Not attempting re-org due to strong head"; - "head" => ?canonical_head, - "head_weight" => ?proposer_head.canonical_head_weight, - "re_org_weight" => ?proposer_head.re_org_weight_threshold, - ); + cache_hit = false; } - } else if !head_late { - debug!( - self.log, - "Not attempting re-org of timely block"; - "head" => ?canonical_head, - ); - } else if !proposing_on_time { - debug!( - self.log, - "Not attempting re-org due to insufficient time"; - "head" => ?canonical_head, - ) - } else { - debug!( - self.log, - "Not attempting re-org due to slot distance"; - "head" => ?canonical_head, - ); } + debug!( + self.log, + "Not attempting re-org"; + "head" => ?canonical_head, + "head_weight" => ?proposer_head.canonical_head_weight, + "re_org_weight" => ?proposer_head.re_org_weight_threshold, + "head_late" => head_late, + "proposing_on_time" => proposing_on_time, + "single_slot" => proposer_head.is_single_slot_re_org, + "ffg_competitive" => proposer_head.ffg_competitive, + "cache_hit" => cache_hit, + "shuffling_stable" => proposer_head.shuffling_stable, + ); } Ok(None) @@ -3362,8 +3340,8 @@ impl BeaconChain { } // Load details of the head block and its parent from fork choice. - let (head_slot, parent_root, parent_slot) = { - let nodes = self + let (head_node, parent_node) = { + let mut nodes = self .canonical_head .fork_choice_read_lock() .proto_array() @@ -3377,29 +3355,37 @@ impl BeaconChain { return Ok(false); } - (nodes[0].slot, nodes[1].root, nodes[1].slot) + let parent = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; + let head = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; + (head, parent) }; // The slot of our potential re-org block is always 1 greater than the head block because we // only attempt single-slot re-orgs. - let re_org_block_slot = head_slot + 1; + let re_org_block_slot = head_node.slot + 1; // Suppress only during the head block's slot, or at most until the end of the next slot. // If our proposal fails entirely we will attest to the wrong head during // `re_org_block_slot` and only re-align with the canonical chain 500ms before the start of // the next slot (i.e. `head_slot + 2`). - let current_slot_ok = head_slot == current_slot || re_org_block_slot == current_slot; + let current_slot_ok = head_node.slot == current_slot || re_org_block_slot == current_slot; // Only attempt single slot re-orgs, and not at epoch boundaries. - let block_slot_ok = - parent_slot + 1 == head_slot && re_org_block_slot % T::EthSpec::slots_per_epoch() != 0; + let block_slot_ok = parent_node.slot + 1 == head_node.slot + && re_org_block_slot % T::EthSpec::slots_per_epoch() != 0; + + // Only attempt re-orgs with competitive FFG information. + let ffg_competitive = parent_node.unrealized_justified_checkpoint + == head_node.unrealized_justified_checkpoint + && parent_node.unrealized_finalized_checkpoint + == head_node.unrealized_finalized_checkpoint; // Check that this node has a proposer prepared to execute a re-org. let prepared_for_re_org = self .execution_layer .as_ref() .ok_or(Error::ExecutionLayerMissing)? - .payload_attributes(re_org_block_slot, parent_root) + .payload_attributes(re_org_block_slot, parent_node.root) .await .is_some(); @@ -3409,10 +3395,13 @@ impl BeaconChain { // attestations for it. We also can't dequeue attestations for the block during the // current slot, which would be necessary for determining its weight. let head_block_late = - self.block_observed_after_attestation_deadline(head_block_root, head_slot); + self.block_observed_after_attestation_deadline(head_block_root, head_node.slot); - let might_re_org = - current_slot_ok && block_slot_ok && prepared_for_re_org && head_block_late; + let might_re_org = current_slot_ok + && block_slot_ok + && ffg_competitive + && prepared_for_re_org + && head_block_late; Ok(might_re_org) } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 6b74bcd1fa6..49adb2fe012 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -204,6 +204,7 @@ pub enum BeaconChainError { MissingPersistedForkChoice, CommitteeCacheWait(crossbeam_channel::RecvError), MaxCommitteePromises(usize), + SuppressForkChoiceError, } easy_from_to!(SlotProcessingError, BeaconChainError); diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 27ba5a8e7f6..618327fcd94 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -3,6 +3,7 @@ use crate::common::*; use beacon_chain::test_utils::{AttestationStrategy, BlockStrategy}; use eth2::types::DepositContractData; use slot_clock::SlotClock; +use state_processing::state_advance::complete_state_advance; use tree_hash::TreeHash; use types::{EthSpec, FullPayload, MainnetEthSpec, Slot}; @@ -37,17 +38,38 @@ async fn deposit_contract_custom_network() { // Test that the beacon node will try to perform proposer boost re-orgs on late blocks when // configured. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -pub async fn proposer_boost_re_org_success() { +pub async fn proposer_boost_re_org_zero_weight() { + proposer_boost_re_org_test(Slot::new(30), None, Some(10), true).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_epoch_boundary() { + proposer_boost_re_org_test(Slot::new(31), None, Some(10), false).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_bad_ffg() { + proposer_boost_re_org_test(Slot::new(64 + 22), None, Some(10), false).await +} + +pub async fn proposer_boost_re_org_test( + head_slot: Slot, + num_head_votes: Option, + re_org_threshold: Option, + should_re_org: bool, +) { + assert!(head_slot > 0); + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing // `validator_count // 32`. let validator_count = 32; - let num_initial: u64 = 31; + let num_initial = head_slot.as_u64() - 1; let tester = InteractiveTester::::new_with_mutator( None, validator_count, - Some(Box::new(|builder| { - builder.proposer_re_org_threshold(Some(10)) + Some(Box::new(move |builder| { + builder.proposer_re_org_threshold(re_org_threshold) })), ) .await; @@ -77,12 +99,13 @@ pub async fn proposer_boost_re_org_success() { let state_a = harness.get_current_state(); // Produce block B and process it halfway through the slot. - let (block_b, state_b) = harness.make_block(state_a.clone(), slot_b).await; + let (block_b, mut state_b) = harness.make_block(state_a.clone(), slot_b).await; + let block_b_root = block_b.canonical_root(); let obs_time = slot_clock.start_of(slot_b).unwrap() + slot_clock.slot_duration() / 2; slot_clock.set_current_time(obs_time); harness.chain.block_times_cache.write().set_time_observed( - block_b.canonical_root(), + block_b_root, slot_b, obs_time, None, @@ -90,9 +113,27 @@ pub async fn proposer_boost_re_org_success() { ); harness.process_block_result(block_b).await.unwrap(); + // Add attestations to block B. + /* FIXME(sproul): implement attestations + if let Some(num_head_votes) = num_head_votes { + harness.attest_block( + &state_b, + state_b.canonical_root(), + block_b_root, + &block_b, + &[] + ) + } + */ + // Produce block C. - harness.advance_slot(); - harness.chain.per_slot_task().await; + while harness.get_current_slot() != slot_c { + harness.advance_slot(); + harness.chain.per_slot_task().await; + } + + // Advance state_b so we can get the proposer. + complete_state_advance(&mut state_b, None, slot_c, &harness.chain.spec).unwrap(); let proposer_index = state_b .get_beacon_proposer_index(slot_c, &harness.chain.spec) @@ -108,10 +149,15 @@ pub async fn proposer_boost_re_org_success() { .data; let block_c = harness.sign_beacon_block(unsigned_block_c, &state_b); - // Block C should build on A. - assert_eq!(block_c.parent_root(), block_a_root); + if should_re_org { + // Block C should build on A. + assert_eq!(block_c.parent_root(), block_a_root); + } else { + // Block C should build on B. + assert_eq!(block_c.parent_root(), block_b_root); + } - // Applying block C should cause a re-org from B to C. + // Applying block C should cause it to become head regardless (re-org or continuation). let block_root_c = harness.process_block_result(block_c).await.unwrap().into(); assert_eq!(harness.head_block_root(), block_root_c); } diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index fcda9fb9a51..73a3d07efc5 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -567,6 +567,7 @@ where self.proto_array .get_proposer_head::( + current_slot, self.fc_store.justified_balances(), canonical_head, re_org_threshold, diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index be8e148e282..07a82a3c30e 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -183,6 +183,12 @@ pub struct ProposerHead { pub canonical_head_weight: Option, /// The computed fraction of the active committee balance below which we can re-org. pub re_org_weight_threshold: Option, + /// Is this is a single slot re-org? + pub is_single_slot_re_org: bool, + /// Is the proposer head's FFG information competitive with the head to be re-orged? + pub ffg_competitive: bool, + /// Is the re-org block off an epoch boundary where the proposer shuffling could change? + pub shuffling_stable: bool, } #[derive(PartialEq)] @@ -337,6 +343,7 @@ impl ProtoArrayForkChoice { pub fn get_proposer_head( &self, + current_slot: Slot, justified_state_balances: &[u64], canonical_head: Hash256, re_org_vote_fraction: u64, @@ -352,8 +359,25 @@ impl ProtoArrayForkChoice { let head_node = nodes[0]; let parent_node = nodes[1]; - // Re-org conditions. - let is_single_slot_re_org = parent_node.slot + 1 == head_node.slot; + // Only re-org a single slot. This prevents cascading failures during asynchrony. + let is_single_slot_re_org = + parent_node.slot + 1 == head_node.slot && head_node.slot + 1 == current_slot; + + // Do not re-org one the first slot of an epoch because this is liable to change the + // shuffling and rob us of a proposal entirely. A more sophisticated check could be + // done here, but we're prioristing speed and simplicity over precision. + let shuffling_stable = current_slot % E::slots_per_epoch() != 0; + + // Only re-org if the new head will be competitive with the current head's justification and + // finalization. In lieu of computing new justification and finalization for our re-org + // block that hasn't been created yet, just check if the parent we would build on is + // competitive with the head. + let ffg_competitive = parent_node.unrealized_justified_checkpoint + == head_node.unrealized_justified_checkpoint + && parent_node.unrealized_finalized_checkpoint + == head_node.unrealized_finalized_checkpoint; + + // Only re-org if the head's weight is less than the configured committee fraction. let re_org_weight_threshold = calculate_proposer_boost::(justified_state_balances, re_org_vote_fraction) .ok_or_else(|| { @@ -364,12 +388,17 @@ impl ProtoArrayForkChoice { .map_err(|e| format!("overflow calculating head weight: {:?}", e))?; let is_weak_head = canonical_head_weight < re_org_weight_threshold; - let re_org_head = (is_single_slot_re_org && is_weak_head).then(|| parent_node.root); + let re_org_head = + (is_single_slot_re_org && shuffling_stable && ffg_competitive && is_weak_head) + .then(|| parent_node.root); Ok(ProposerHead { re_org_head, canonical_head_weight: Some(canonical_head_weight), re_org_weight_threshold: Some(re_org_weight_threshold), + is_single_slot_re_org, + ffg_competitive, + shuffling_stable, }) } From 4728ac341e288b43c47998165b0cb8ab610bc364 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 4 Oct 2022 17:07:18 +1100 Subject: [PATCH 11/37] Participation check, justified balance abstraction --- Cargo.lock | 1 + beacon_node/beacon_chain/src/beacon_chain.rs | 153 ++++++++++-------- .../src/beacon_fork_choice_store.rs | 53 +++--- beacon_node/beacon_chain/src/builder.rs | 9 +- beacon_node/beacon_chain/src/chain_config.rs | 13 +- beacon_node/beacon_chain/src/fork_revert.rs | 3 +- .../src/schema_change/migration_schema_v7.rs | 8 +- .../http_api/tests/interactive_tests.rs | 7 +- beacon_node/src/config.rs | 3 +- consensus/fork_choice/src/fork_choice.rs | 27 +++- .../fork_choice/src/fork_choice_store.rs | 3 +- consensus/fork_choice/tests/tests.rs | 8 +- consensus/proto_array/Cargo.toml | 1 + consensus/proto_array/src/error.rs | 8 + .../src/fork_choice_test_definition.rs | 17 +- .../proto_array/src/justified_balances.rs | 62 +++++++ consensus/proto_array/src/lib.rs | 5 +- consensus/proto_array/src/proto_array.rs | 35 ++-- .../src/proto_array_fork_choice.rs | 80 +++++---- consensus/proto_array/src/ssz_container.rs | 18 ++- lighthouse/tests/beacon_node.rs | 2 +- 21 files changed, 331 insertions(+), 185 deletions(-) create mode 100644 consensus/proto_array/src/justified_balances.rs diff --git a/Cargo.lock b/Cargo.lock index 4ca2739d14a..8df472f54f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5017,6 +5017,7 @@ version = "0.2.0" dependencies = [ "eth2_ssz", "eth2_ssz_derive", + "safe_arith", "serde", "serde_derive", "serde_yaml", diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 8f09af24749..192b798f716 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3187,8 +3187,7 @@ impl BeaconChain { }; let (state, state_root_opt) = if head_slot < slot { // Attempt an aggressive re-org if configured and the conditions are right. - if let Some(re_org_state) = - self.get_state_for_re_org(slot, head_slot, head_block_root)? + if let Some(re_org_state) = self.get_state_for_re_org(slot, head_slot, head_block_root) { info!( self.log, @@ -3241,84 +3240,102 @@ impl BeaconChain { /// Fetch the beacon state to use for producing a block if a 1-slot proposer re-org is viable. /// - /// This function will return `Ok(None)` if proposer re-orgs are disabled. + /// This function will return `None` if proposer re-orgs are disabled. fn get_state_for_re_org( &self, slot: Slot, head_slot: Slot, canonical_head: Hash256, - ) -> Result>, BlockProductionError> { - if let Some(re_org_threshold) = self.config.re_org_threshold { - if self.spec.proposer_score_boost.is_none() { + ) -> Option> { + let re_org_threshold = self.config.re_org_threshold?; + + if self.spec.proposer_score_boost.is_none() { + warn!( + self.log, + "Ignoring proposer re-org configuration"; + "reason" => "this network does not have proposer boost enabled" + ); + return None; + } + + let slot_delay = self + .slot_clock + .seconds_from_current_slot_start(self.spec.seconds_per_slot) + .or_else(|| { warn!( self.log, - "Ignoring proposer re-org configuration"; - "reason" => "this network does not have proposer boost enabled" + "Not attempting re-org"; + "error" => "unable to read slot clock" ); - return Ok(None); - } - - let slot_delay = self - .slot_clock - .seconds_from_current_slot_start(self.spec.seconds_per_slot) - .ok_or(BlockProductionError::UnableToReadSlot)?; + None + })?; - // Attempt a proposer re-org if: - // - // 1. It seems we have time to propagate and still receive the proposer boost. - // 2. The current head block was seen late. - // 3. The `get_proposer_head` conditions from fork choice pass. - let proposing_on_time = slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot); - let head_late = - self.block_observed_after_attestation_deadline(canonical_head, head_slot); - let mut proposer_head = Default::default(); - let mut cache_hit = true; - - if proposing_on_time && head_late { - // Is the current head weak and appropriate for re-orging? - proposer_head = self - .canonical_head - .fork_choice_write_lock() - .get_proposer_head(slot, canonical_head, re_org_threshold, &self.spec)?; - - if let Some(re_org_head) = proposer_head.re_org_head { - // Only attempt a re-org if we hit the snapshot cache. - if let Some(pre_state) = self - .snapshot_cache - .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .and_then(|snapshot_cache| { - snapshot_cache.get_state_for_block_production(re_org_head) - }) - { - debug!( - self.log, - "Attempting re-org due to weak head"; - "head" => ?canonical_head, - "re_org_head" => ?re_org_head, - "head_weight" => ?proposer_head.canonical_head_weight, - "re_org_weight" => ?proposer_head.re_org_weight_threshold, - ); - return Ok(Some(pre_state)); - } - cache_hit = false; + // Attempt a proposer re-org if: + // + // 1. It seems we have time to propagate and still receive the proposer boost. + // 2. The current head block was seen late. + // 3. The `get_proposer_head` conditions from fork choice pass. + let proposing_on_time = slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot); + let head_late = self.block_observed_after_attestation_deadline(canonical_head, head_slot); + let mut proposer_head = Default::default(); + let mut cache_hit = true; + + if proposing_on_time && head_late { + // Is the current head weak and appropriate for re-orging? + proposer_head = self + .canonical_head + .fork_choice_read_lock() + .get_proposer_head( + slot, + canonical_head, + re_org_threshold, + self.config.re_org_participation_threshold, + ) + .map_err(|e| { + warn!( + self.log, + "Not attempting re-org"; + "error" => ?e, + ); + }) + .ok()?; + + if let Some(re_org_head) = proposer_head.re_org_head { + // Only attempt a re-org if we hit the snapshot cache. + if let Some(pre_state) = self + .snapshot_cache + .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) + .and_then(|snapshot_cache| { + snapshot_cache.get_state_for_block_production(re_org_head) + }) + { + info!( + self.log, + "Attempting re-org due to weak head"; + "head" => ?canonical_head, + "re_org_head" => ?re_org_head, + "head_weight" => ?proposer_head.canonical_head_weight, + "re_org_weight" => ?proposer_head.re_org_weight_threshold, + ); + return Some(pre_state); } + cache_hit = false; } - debug!( - self.log, - "Not attempting re-org"; - "head" => ?canonical_head, - "head_weight" => ?proposer_head.canonical_head_weight, - "re_org_weight" => ?proposer_head.re_org_weight_threshold, - "head_late" => head_late, - "proposing_on_time" => proposing_on_time, - "single_slot" => proposer_head.is_single_slot_re_org, - "ffg_competitive" => proposer_head.ffg_competitive, - "cache_hit" => cache_hit, - "shuffling_stable" => proposer_head.shuffling_stable, - ); } - - Ok(None) + debug!( + self.log, + "Not attempting re-org"; + "head" => ?canonical_head, + "head_weight" => ?proposer_head.canonical_head_weight, + "re_org_weight" => ?proposer_head.re_org_weight_threshold, + "head_late" => head_late, + "proposing_on_time" => proposing_on_time, + "single_slot" => proposer_head.is_single_slot_re_org, + "ffg_competitive" => proposer_head.ffg_competitive, + "cache_hit" => cache_hit, + "shuffling_stable" => proposer_head.shuffling_stable, + ); + None } /// Determine whether a fork choice update to the execution layer should be suppressed. diff --git a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs index 4f6003fda1b..a410398e864 100644 --- a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs +++ b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs @@ -7,6 +7,8 @@ use crate::{metrics, BeaconSnapshot}; use derivative::Derivative; use fork_choice::ForkChoiceStore; +use proto_array::JustifiedBalances; +use safe_arith::ArithError; use ssz_derive::{Decode, Encode}; use std::collections::BTreeSet; use std::marker::PhantomData; @@ -31,6 +33,7 @@ pub enum Error { MissingState(Hash256), InvalidPersistedBytes(ssz::DecodeError), BeaconStateError(BeaconStateError), + Arith(ArithError), } impl From for Error { @@ -39,27 +42,15 @@ impl From for Error { } } +impl From for Error { + fn from(e: ArithError) -> Self { + Error::Arith(e) + } +} + /// The number of validator balance sets that are cached within `BalancesCache`. const MAX_BALANCE_CACHE_SIZE: usize = 4; -/// Returns the effective balances for every validator in the given `state`. -/// -/// Any validator who is not active in the epoch of the given `state` is assigned a balance of -/// zero. -pub fn get_effective_balances(state: &BeaconState) -> Vec { - state - .validators() - .iter() - .map(|validator| { - if validator.is_active_at(state.current_epoch()) { - validator.effective_balance - } else { - 0 - } - }) - .collect() -} - #[superstruct( variants(V1, V8), variant_attributes(derive(PartialEq, Clone, Debug, Encode, Decode)), @@ -115,7 +106,7 @@ impl BalancesCache { let item = CacheItem { block_root: epoch_boundary_root, epoch, - balances: get_effective_balances(state), + balances: JustifiedBalances::from_justified_state(state)?.effective_balances, }; if self.items.len() == MAX_BALANCE_CACHE_SIZE { @@ -154,7 +145,7 @@ pub struct BeaconForkChoiceStore, Cold: ItemStore< time: Slot, finalized_checkpoint: Checkpoint, justified_checkpoint: Checkpoint, - justified_balances: Vec, + justified_balances: JustifiedBalances, best_justified_checkpoint: Checkpoint, unrealized_justified_checkpoint: Checkpoint, unrealized_finalized_checkpoint: Checkpoint, @@ -183,7 +174,7 @@ where pub fn get_forkchoice_store( store: Arc>, anchor: &BeaconSnapshot, - ) -> Self { + ) -> Result { let anchor_state = &anchor.beacon_state; let mut anchor_block_header = anchor_state.latest_block_header().clone(); if anchor_block_header.state_root == Hash256::zero() { @@ -196,13 +187,14 @@ where root: anchor_root, }; let finalized_checkpoint = justified_checkpoint; + let justified_balances = JustifiedBalances::from_justified_state(&anchor_state)?; - Self { + Ok(Self { store, balances_cache: <_>::default(), time: anchor_state.slot(), justified_checkpoint, - justified_balances: anchor_state.balances().clone().into(), + justified_balances, finalized_checkpoint, best_justified_checkpoint: justified_checkpoint, unrealized_justified_checkpoint: justified_checkpoint, @@ -210,7 +202,7 @@ where proposer_boost_root: Hash256::zero(), equivocating_indices: BTreeSet::new(), _phantom: PhantomData, - } + }) } /// Save the current state of `Self` to a `PersistedForkChoiceStore` which can be stored to the @@ -221,7 +213,7 @@ where time: self.time, finalized_checkpoint: self.finalized_checkpoint, justified_checkpoint: self.justified_checkpoint, - justified_balances: self.justified_balances.clone(), + justified_balances: self.justified_balances.effective_balances.clone(), best_justified_checkpoint: self.best_justified_checkpoint, unrealized_justified_checkpoint: self.unrealized_justified_checkpoint, unrealized_finalized_checkpoint: self.unrealized_finalized_checkpoint, @@ -235,13 +227,15 @@ where persisted: PersistedForkChoiceStore, store: Arc>, ) -> Result { + let justified_balances = + JustifiedBalances::from_effective_balances(persisted.justified_balances)?; Ok(Self { store, balances_cache: persisted.balances_cache, time: persisted.time, finalized_checkpoint: persisted.finalized_checkpoint, justified_checkpoint: persisted.justified_checkpoint, - justified_balances: persisted.justified_balances, + justified_balances, best_justified_checkpoint: persisted.best_justified_checkpoint, unrealized_justified_checkpoint: persisted.unrealized_justified_checkpoint, unrealized_finalized_checkpoint: persisted.unrealized_finalized_checkpoint, @@ -281,7 +275,7 @@ where &self.justified_checkpoint } - fn justified_balances(&self) -> &[u64] { + fn justified_balances(&self) -> &JustifiedBalances { &self.justified_balances } @@ -316,8 +310,9 @@ where self.justified_checkpoint.root, self.justified_checkpoint.epoch, ) { + // NOTE: could avoid this re-calculation by introducing a `PersistedCacheItem`. metrics::inc_counter(&metrics::BALANCES_CACHE_HITS); - self.justified_balances = balances; + self.justified_balances = JustifiedBalances::from_effective_balances(balances)?; } else { metrics::inc_counter(&metrics::BALANCES_CACHE_MISSES); let justified_block = self @@ -334,7 +329,7 @@ where .map_err(Error::FailedToReadState)? .ok_or_else(|| Error::MissingState(justified_block.state_root()))?; - self.justified_balances = get_effective_balances(&state); + self.justified_balances = JustifiedBalances::from_justified_state(&state)?; } Ok(()) diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index e9c06cfa1ef..146e1f6bde0 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -21,6 +21,7 @@ use fork_choice::{ForkChoice, ResetPayloadStatuses}; use futures::channel::mpsc::Sender; use operation_pool::{OperationPool, PersistedOperationPool}; use parking_lot::RwLock; +use proto_array::ReOrgThreshold; use slasher::Slasher; use slog::{crit, error, info, Logger}; use slot_clock::{SlotClock, TestingSlotClock}; @@ -159,7 +160,7 @@ where } /// Sets the proposer re-org threshold. - pub fn proposer_re_org_threshold(mut self, threshold: Option) -> Self { + pub fn proposer_re_org_threshold(mut self, threshold: Option) -> Self { self.chain_config.re_org_threshold = threshold; self } @@ -363,7 +364,8 @@ where let (genesis, updated_builder) = self.set_genesis_state(beacon_state)?; self = updated_builder; - let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis); + let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis) + .map_err(|e| format!("Unable to initialize fork choice store: {e:?}"))?; let current_slot = None; let fork_choice = ForkChoice::from_anchor( @@ -481,7 +483,8 @@ where beacon_state: weak_subj_state, }; - let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &snapshot); + let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &snapshot) + .map_err(|e| format!("Unable to initialize fork choice store: {e:?}"))?; let current_slot = Some(snapshot.beacon_block.slot()); let fork_choice = ForkChoice::from_anchor( diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index fee5d45c728..a1d35eea016 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -1,8 +1,10 @@ -pub use proto_array::CountUnrealizedFull; +pub use proto_array::{CountUnrealizedFull, ParticipationThreshold, ReOrgThreshold}; use serde_derive::{Deserialize, Serialize}; use types::Checkpoint; -pub const DEFAULT_RE_ORG_THRESHOLD: u64 = 10; +pub const DEFAULT_RE_ORG_THRESHOLD: ReOrgThreshold = ReOrgThreshold(10); +pub const DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD: ParticipationThreshold = + ParticipationThreshold(80); pub const DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT: u64 = 250; #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] @@ -22,8 +24,10 @@ pub struct ChainConfig { pub enable_lock_timeouts: bool, /// The max size of a message that can be sent over the network. pub max_network_size: usize, - /// Maximum percentage of weight at which to attempt re-orging the canonical head. - pub re_org_threshold: Option, + /// Maximum percentage of committee weight at which to attempt re-orging the canonical head. + pub re_org_threshold: Option, + /// Minimum participation at which a proposer re-org should be attempted. + pub re_org_participation_threshold: ParticipationThreshold, /// Number of milliseconds to wait for fork choice before proposing a block. /// /// If set to 0 then block proposal will not wait for fork choice at all. @@ -59,6 +63,7 @@ impl Default for ChainConfig { enable_lock_timeouts: true, max_network_size: 10 * 1_048_576, // 10M re_org_threshold: None, + re_org_participation_threshold: DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD, fork_choice_before_proposal_timeout_ms: DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT, // Builder fallback configs that are set in `clap` will override these. builder_fallback_skips: 3, diff --git a/beacon_node/beacon_chain/src/fork_revert.rs b/beacon_node/beacon_chain/src/fork_revert.rs index 654b2713b1c..51c3904f412 100644 --- a/beacon_node/beacon_chain/src/fork_revert.rs +++ b/beacon_node/beacon_chain/src/fork_revert.rs @@ -146,7 +146,8 @@ pub fn reset_fork_choice_to_finalization, Cold: It beacon_state: finalized_state, }; - let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), &finalized_snapshot); + let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store.clone(), &finalized_snapshot) + .map_err(|e| format!("Unable to reset fork choice store for revert: {e:?}"))?; let mut fork_choice = ForkChoice::from_anchor( fc_store, diff --git a/beacon_node/beacon_chain/src/schema_change/migration_schema_v7.rs b/beacon_node/beacon_chain/src/schema_change/migration_schema_v7.rs index 4a9a78db7b6..e3a766f698c 100644 --- a/beacon_node/beacon_chain/src/schema_change/migration_schema_v7.rs +++ b/beacon_node/beacon_chain/src/schema_change/migration_schema_v7.rs @@ -43,7 +43,8 @@ pub(crate) fn update_with_reinitialized_fork_choice( beacon_block_root: anchor_block_root, beacon_state: anchor_state, }; - let store = BeaconForkChoiceStore::get_forkchoice_store(db, &snapshot); + let store = BeaconForkChoiceStore::get_forkchoice_store(db, &snapshot) + .map_err(|e| format!("Unable to initialize fork choice store: {e:?}"))?; let fork_choice = ForkChoice::from_anchor( store, anchor_block_root, @@ -91,8 +92,9 @@ pub(crate) fn update_fork_choice( let ssz_container_v10: SszContainerV10 = ssz_container_v7.into(); let ssz_container: SszContainer = ssz_container_v10.into(); // `CountUnrealizedFull::default()` represents the count-unrealized-full config which will be overwritten on startup. - let mut fork_choice: ProtoArrayForkChoice = - (ssz_container, CountUnrealizedFull::default()).into(); + let mut fork_choice: ProtoArrayForkChoice = (ssz_container, CountUnrealizedFull::default()) + .try_into() + .map_err(|e| StoreError::SchemaMigrationError(format!("{e:?}")))?; update_checkpoints::(finalized_checkpoint.root, &nodes_v6, &mut fork_choice, db) .map_err(StoreError::SchemaMigrationError)?; diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 618327fcd94..5443f6c9b82 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -1,6 +1,9 @@ //! Generic tests that make use of the (newer) `InteractiveApiTester` use crate::common::*; -use beacon_chain::test_utils::{AttestationStrategy, BlockStrategy}; +use beacon_chain::{ + chain_config::ReOrgThreshold, + test_utils::{AttestationStrategy, BlockStrategy}, +}; use eth2::types::DepositContractData; use slot_clock::SlotClock; use state_processing::state_advance::complete_state_advance; @@ -69,7 +72,7 @@ pub async fn proposer_boost_re_org_test( None, validator_count, Some(Box::new(move |builder| { - builder.proposer_re_org_threshold(re_org_threshold) + builder.proposer_re_org_threshold(re_org_threshold.map(ReOrgThreshold)) })), ) .await; diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 0700ab85099..97c48e03aaa 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1,4 +1,4 @@ -use beacon_chain::chain_config::DEFAULT_RE_ORG_THRESHOLD; +use beacon_chain::chain_config::{ReOrgThreshold, DEFAULT_RE_ORG_THRESHOLD}; use clap::ArgMatches; use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG; use client::{ClientConfig, ClientGenesis}; @@ -650,6 +650,7 @@ pub fn get_config( if enable_re_orgs { client_config.chain.re_org_threshold = Some( clap_utils::parse_optional(cli_args, "proposer-re-org-fraction")? + .map(ReOrgThreshold) .unwrap_or(DEFAULT_RE_ORG_THRESHOLD), ); } else { diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 73a3d07efc5..6ac1bcb8707 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -1,6 +1,7 @@ use crate::{ForkChoiceStore, InvalidationOperation}; use proto_array::{ - Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, ProposerHead, ProtoArrayForkChoice, + Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, ParticipationThreshold, + ProposerHead, ProtoArrayForkChoice, ReOrgThreshold, }; use slog::{crit, debug, warn, Logger}; use ssz_derive::{Decode, Encode}; @@ -59,6 +60,10 @@ pub enum Error { MissingFinalizedBlock { finalized_checkpoint: Checkpoint, }, + WrongSlotForGetProposerHead { + current_slot: Slot, + fc_store_slot: Slot, + }, UnrealizedVoteProcessing(state_processing::EpochProcessingError), ParticipationCacheBuild(BeaconStateError), ValidatorStatuses(BeaconStateError), @@ -555,15 +560,22 @@ where } pub fn get_proposer_head( - &mut self, + &self, current_slot: Slot, canonical_head: Hash256, - re_org_threshold: u64, - spec: &ChainSpec, + re_org_threshold: ReOrgThreshold, + participation_threshold: ParticipationThreshold, ) -> Result> { - // Calling `update_time` is essential, as it needs to dequeue attestations from the previous - // slot so we can see how many attesters voted for the canonical head. - self.update_time(current_slot, spec)?; + // Ensure that fork choice has already been updated for the current slot. This prevents + // us from having to take a write lock or do any dequeueing of attestations in this + // function. + let fc_store_slot = self.fc_store.get_current_slot(); + if current_slot != fc_store_slot { + return Err(Error::WrongSlotForGetProposerHead { + current_slot, + fc_store_slot, + }); + } self.proto_array .get_proposer_head::( @@ -571,6 +583,7 @@ where self.fc_store.justified_balances(), canonical_head, re_org_threshold, + participation_threshold, ) .map_err(Into::into) } diff --git a/consensus/fork_choice/src/fork_choice_store.rs b/consensus/fork_choice/src/fork_choice_store.rs index 9604e254754..60c58859ed8 100644 --- a/consensus/fork_choice/src/fork_choice_store.rs +++ b/consensus/fork_choice/src/fork_choice_store.rs @@ -1,3 +1,4 @@ +use proto_array::JustifiedBalances; use std::collections::BTreeSet; use std::fmt::Debug; use types::{BeaconBlockRef, BeaconState, Checkpoint, EthSpec, ExecPayload, Hash256, Slot}; @@ -44,7 +45,7 @@ pub trait ForkChoiceStore: Sized { fn justified_checkpoint(&self) -> &Checkpoint; /// Returns balances from the `state` identified by `justified_checkpoint.root`. - fn justified_balances(&self) -> &[u64]; + fn justified_balances(&self) -> &JustifiedBalances; /// Returns the `best_justified_checkpoint`. fn best_justified_checkpoint(&self) -> &Checkpoint; diff --git a/consensus/fork_choice/tests/tests.rs b/consensus/fork_choice/tests/tests.rs index 850f7c4a120..00bd1f763dc 100644 --- a/consensus/fork_choice/tests/tests.rs +++ b/consensus/fork_choice/tests/tests.rs @@ -378,9 +378,13 @@ impl ForkChoiceTest { assert_eq!( &balances[..], - fc.fc_store().justified_balances(), + &fc.fc_store().justified_balances().effective_balances, "balances should match" - ) + ); + assert_eq!( + balances.iter().sum::(), + fc.fc_store().justified_balances().total_effective_balance + ); } /// Returns an attestation that is valid for some slot in the given `chain`. diff --git a/consensus/proto_array/Cargo.toml b/consensus/proto_array/Cargo.toml index ad79ecc1e6b..dfab6fda567 100644 --- a/consensus/proto_array/Cargo.toml +++ b/consensus/proto_array/Cargo.toml @@ -15,3 +15,4 @@ eth2_ssz_derive = "0.3.0" serde = "1.0.116" serde_derive = "1.0.116" serde_yaml = "0.8.13" +safe_arith = { path = "../safe_arith" } diff --git a/consensus/proto_array/src/error.rs b/consensus/proto_array/src/error.rs index 6f715874456..d2fdad56a77 100644 --- a/consensus/proto_array/src/error.rs +++ b/consensus/proto_array/src/error.rs @@ -1,3 +1,4 @@ +use safe_arith::ArithError; use types::{Checkpoint, Epoch, ExecutionBlockHash, Hash256, Slot}; #[derive(Clone, PartialEq, Debug)] @@ -49,6 +50,13 @@ pub enum Error { block_root: Hash256, parent_root: Hash256, }, + Arith(ArithError), +} + +impl From for Error { + fn from(e: ArithError) -> Self { + Error::Arith(e) + } } #[derive(Clone, PartialEq, Debug)] diff --git a/consensus/proto_array/src/fork_choice_test_definition.rs b/consensus/proto_array/src/fork_choice_test_definition.rs index ba6f3170dc1..035fb799eea 100644 --- a/consensus/proto_array/src/fork_choice_test_definition.rs +++ b/consensus/proto_array/src/fork_choice_test_definition.rs @@ -5,7 +5,7 @@ mod votes; use crate::proto_array::CountUnrealizedFull; use crate::proto_array_fork_choice::{Block, ExecutionStatus, ProtoArrayForkChoice}; -use crate::InvalidationOperation; +use crate::{InvalidationOperation, JustifiedBalances}; use serde_derive::{Deserialize, Serialize}; use std::collections::BTreeSet; use types::{ @@ -101,11 +101,14 @@ impl ForkChoiceTestDefinition { justified_state_balances, expected_head, } => { + let justified_balances = + JustifiedBalances::from_effective_balances(justified_state_balances) + .unwrap(); let head = fork_choice .find_head::( justified_checkpoint, finalized_checkpoint, - &justified_state_balances, + &justified_balances, Hash256::zero(), &equivocating_indices, Slot::new(0), @@ -129,11 +132,14 @@ impl ForkChoiceTestDefinition { expected_head, proposer_boost_root, } => { + let justified_balances = + JustifiedBalances::from_effective_balances(justified_state_balances) + .unwrap(); let head = fork_choice .find_head::( justified_checkpoint, finalized_checkpoint, - &justified_state_balances, + &justified_balances, proposer_boost_root, &equivocating_indices, Slot::new(0), @@ -155,10 +161,13 @@ impl ForkChoiceTestDefinition { finalized_checkpoint, justified_state_balances, } => { + let justified_balances = + JustifiedBalances::from_effective_balances(justified_state_balances) + .unwrap(); let result = fork_choice.find_head::( justified_checkpoint, finalized_checkpoint, - &justified_state_balances, + &justified_balances, Hash256::zero(), &equivocating_indices, Slot::new(0), diff --git a/consensus/proto_array/src/justified_balances.rs b/consensus/proto_array/src/justified_balances.rs new file mode 100644 index 00000000000..75f6c2f7c80 --- /dev/null +++ b/consensus/proto_array/src/justified_balances.rs @@ -0,0 +1,62 @@ +use safe_arith::{ArithError, SafeArith}; +use types::{BeaconState, EthSpec}; + +#[derive(Debug, PartialEq, Clone, Default)] +pub struct JustifiedBalances { + /// The effective balances for every validator in a given justified state. + /// + /// Any validator who is not active in the epoch of the justified state is assigned a balance of + /// zero. + pub effective_balances: Vec, + /// The sum of `self.effective_balances`. + pub total_effective_balance: u64, + /// The number of active validators included in `self.effective_balances`. + pub num_active_validators: u64, +} + +impl JustifiedBalances { + pub fn from_justified_state(state: &BeaconState) -> Result { + let current_epoch = state.current_epoch(); + let mut total_effective_balance = 0u64; + let mut num_active_validators = 0u64; + + let effective_balances = state + .validators() + .iter() + .map(|validator| { + if validator.is_active_at(current_epoch) { + total_effective_balance.safe_add_assign(validator.effective_balance)?; + num_active_validators.safe_add_assign(1)?; + + Ok(validator.effective_balance) + } else { + Ok(0) + } + }) + .collect::, _>>()?; + + Ok(Self { + effective_balances, + total_effective_balance, + num_active_validators, + }) + } + + pub fn from_effective_balances(effective_balances: Vec) -> Result { + let mut total_effective_balance = 0; + let mut num_active_validators = 0; + + for &balance in &effective_balances { + if balance != 0 { + total_effective_balance.safe_add_assign(balance)?; + num_active_validators.safe_add_assign(1)?; + } + } + + Ok(Self { + effective_balances, + total_effective_balance, + num_active_validators, + }) + } +} diff --git a/consensus/proto_array/src/lib.rs b/consensus/proto_array/src/lib.rs index 815eee54ff0..a117fcded53 100644 --- a/consensus/proto_array/src/lib.rs +++ b/consensus/proto_array/src/lib.rs @@ -1,12 +1,15 @@ mod error; pub mod fork_choice_test_definition; +mod justified_balances; mod proto_array; mod proto_array_fork_choice; mod ssz_container; +pub use crate::justified_balances::JustifiedBalances; pub use crate::proto_array::{CountUnrealizedFull, InvalidationOperation}; pub use crate::proto_array_fork_choice::{ - Block, ExecutionStatus, ProposerHead, ProtoArrayForkChoice, + Block, ExecutionStatus, ParticipationThreshold, ProposerHead, ProtoArrayForkChoice, + ReOrgThreshold, }; pub use error::Error; diff --git a/consensus/proto_array/src/proto_array.rs b/consensus/proto_array/src/proto_array.rs index 590407d7eb8..3ce4bf65b63 100644 --- a/consensus/proto_array/src/proto_array.rs +++ b/consensus/proto_array/src/proto_array.rs @@ -1,5 +1,5 @@ use crate::error::InvalidBestNodeInfo; -use crate::{error::Error, Block, ExecutionStatus}; +use crate::{error::Error, Block, ExecutionStatus, JustifiedBalances}; use serde_derive::{Deserialize, Serialize}; use ssz::four_byte_option_impl; use ssz::Encode; @@ -169,7 +169,7 @@ impl ProtoArray { mut deltas: Vec, justified_checkpoint: Checkpoint, finalized_checkpoint: Checkpoint, - new_balances: &[u64], + new_justified_balances: &JustifiedBalances, proposer_boost_root: Hash256, current_slot: Slot, spec: &ChainSpec, @@ -242,7 +242,7 @@ impl ProtoArray { && !execution_status_is_invalid { proposer_score = - calculate_proposer_boost::(new_balances, proposer_score_boost) + calculate_proposer_boost::(new_justified_balances, proposer_score_boost) .ok_or(Error::ProposerBoostOverflow(node_index))?; node_delta = node_delta .checked_add(proposer_score as i64) @@ -1006,32 +1006,19 @@ impl ProtoArray { } } -/// A helper method to calculate the proposer boost based on the given `validator_balances`. -/// This does *not* do any verification about whether a boost should or should not be applied. -/// The `validator_balances` array used here is assumed to be structured like the one stored in -/// the `BalancesCache`, where *effective* balances are stored and inactive balances are defaulted -/// to zero. -/// -/// Returns `None` if there is an overflow or underflow when calculating the score. +/// A helper method to calculate the proposer boost based on the given `justified_balances`. /// /// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#get_latest_attesting_balance pub fn calculate_proposer_boost( - validator_balances: &[u64], + justified_balances: &JustifiedBalances, proposer_score_boost: u64, ) -> Option { - let mut total_balance: u64 = 0; - let mut num_validators: u64 = 0; - for &balance in validator_balances { - // We need to filter zero balances here to get an accurate active validator count. - // This is because we default inactive validator balances to zero when creating - // this balances array. - if balance != 0 { - total_balance = total_balance.checked_add(balance)?; - num_validators = num_validators.checked_add(1)?; - } - } - let average_balance = total_balance.checked_div(num_validators)?; - let committee_size = num_validators.checked_div(E::slots_per_epoch())?; + let average_balance = justified_balances + .total_effective_balance + .checked_div(justified_balances.num_active_validators)?; + let committee_size = justified_balances + .num_active_validators + .checked_div(E::slots_per_epoch())?; let committee_weight = committee_size.checked_mul(average_balance)?; committee_weight .checked_mul(proposer_score_boost)? diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index 07a82a3c30e..25e857cae4b 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -1,9 +1,12 @@ -use crate::error::Error; -use crate::proto_array::CountUnrealizedFull; -use crate::proto_array::{ - calculate_proposer_boost, InvalidationOperation, Iter, ProposerBoost, ProtoArray, ProtoNode, +use crate::{ + error::Error, + proto_array::{ + calculate_proposer_boost, CountUnrealizedFull, InvalidationOperation, Iter, ProposerBoost, + ProtoArray, ProtoNode, + }, + ssz_container::SszContainer, + JustifiedBalances, }; -use crate::ssz_container::SszContainer; use serde_derive::{Deserialize, Serialize}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; @@ -189,13 +192,25 @@ pub struct ProposerHead { pub ffg_competitive: bool, /// Is the re-org block off an epoch boundary where the proposer shuffling could change? pub shuffling_stable: bool, + /// Is the chain's participation level sufficiently healthy to justify a re-org? + pub participation_ok: bool, } +/// New-type for the re-org threshold percentage. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ReOrgThreshold(pub u64); + +/// New-type for the participation threshold percentage. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ParticipationThreshold(pub u64); + #[derive(PartialEq)] pub struct ProtoArrayForkChoice { pub(crate) proto_array: ProtoArray, pub(crate) votes: ElasticList, - pub(crate) balances: Vec, + pub(crate) balances: JustifiedBalances, } impl ProtoArrayForkChoice { @@ -244,7 +259,7 @@ impl ProtoArrayForkChoice { Ok(Self { proto_array, votes: ElasticList::default(), - balances: vec![], + balances: JustifiedBalances::default(), }) } @@ -303,21 +318,20 @@ impl ProtoArrayForkChoice { &mut self, justified_checkpoint: Checkpoint, finalized_checkpoint: Checkpoint, - justified_state_balances: &[u64], + justified_state_balances: &JustifiedBalances, proposer_boost_root: Hash256, equivocating_indices: &BTreeSet, current_slot: Slot, spec: &ChainSpec, ) -> Result { let old_balances = &mut self.balances; - let new_balances = justified_state_balances; let deltas = compute_deltas( &self.proto_array.indices, &mut self.votes, - old_balances, - new_balances, + &old_balances.effective_balances, + &new_balances.effective_balances, equivocating_indices, ) .map_err(|e| format!("find_head compute_deltas failed: {:?}", e))?; @@ -334,7 +348,7 @@ impl ProtoArrayForkChoice { ) .map_err(|e| format!("find_head apply_score_changes failed: {:?}", e))?; - *old_balances = new_balances.to_vec(); + *old_balances = new_balances.clone(); self.proto_array .find_head::(&justified_checkpoint.root, current_slot) @@ -344,9 +358,10 @@ impl ProtoArrayForkChoice { pub fn get_proposer_head( &self, current_slot: Slot, - justified_state_balances: &[u64], + justified_balances: &JustifiedBalances, canonical_head: Hash256, - re_org_vote_fraction: u64, + re_org_threshold: ReOrgThreshold, + participation_threshold: ParticipationThreshold, ) -> Result { let nodes = self .proto_array @@ -363,7 +378,7 @@ impl ProtoArrayForkChoice { let is_single_slot_re_org = parent_node.slot + 1 == head_node.slot && head_node.slot + 1 == current_slot; - // Do not re-org one the first slot of an epoch because this is liable to change the + // Do not re-org on the first slot of an epoch because this is liable to change the // shuffling and rob us of a proposal entirely. A more sophisticated check could be // done here, but we're prioristing speed and simplicity over precision. let shuffling_stable = current_slot % E::slots_per_epoch() != 0; @@ -377,20 +392,27 @@ impl ProtoArrayForkChoice { && parent_node.unrealized_finalized_checkpoint == head_node.unrealized_finalized_checkpoint; + // To prevent excessive re-orgs when the chain is struggling, only re-org when participation + // is above the configured threshold. This should not overflow. + let participation_ok = parent_node.weight + >= justified_balances.total_effective_balance * participation_threshold.0 / 100; + // Only re-org if the head's weight is less than the configured committee fraction. let re_org_weight_threshold = - calculate_proposer_boost::(justified_state_balances, re_org_vote_fraction) - .ok_or_else(|| { - "overflow calculating committee weight for proposer boost".to_string() - })?; + calculate_proposer_boost::(justified_balances, re_org_threshold.0).ok_or_else( + || "overflow calculating committee weight for proposer boost".to_string(), + )?; let canonical_head_weight = self - .get_block_unique_weight(canonical_head, justified_state_balances) + .get_block_unique_weight(canonical_head, &justified_balances.effective_balances) .map_err(|e| format!("overflow calculating head weight: {:?}", e))?; let is_weak_head = canonical_head_weight < re_org_weight_threshold; - let re_org_head = - (is_single_slot_re_org && shuffling_stable && ffg_competitive && is_weak_head) - .then(|| parent_node.root); + let re_org_head = (is_single_slot_re_org + && shuffling_stable + && ffg_competitive + && participation_ok + && is_weak_head) + .then(|| parent_node.root); Ok(ProposerHead { re_org_head, @@ -399,6 +421,7 @@ impl ProtoArrayForkChoice { is_single_slot_re_org, ffg_competitive, shuffling_stable, + participation_ok, }) } @@ -479,7 +502,7 @@ impl ProtoArrayForkChoice { if vote.current_root == node.root { // Any voting validator that does not have a balance should be // ignored. This is consistent with `compute_deltas`. - self.balances.get(validator_index) + self.balances.effective_balances.get(validator_index) } else { None } @@ -649,10 +672,11 @@ impl ProtoArrayForkChoice { bytes: &[u8], count_unrealized_full: CountUnrealizedFull, ) -> Result { - SszContainer::from_ssz_bytes(bytes) - .map(|container| (container, count_unrealized_full)) - .map(Into::into) - .map_err(|e| format!("Failed to decode ProtoArrayForkChoice: {:?}", e)) + let container = SszContainer::from_ssz_bytes(bytes) + .map_err(|e| format!("Failed to decode ProtoArrayForkChoice: {:?}", e))?; + (container, count_unrealized_full) + .try_into() + .map_err(|e| format!("Failed to initialize ProtoArrayForkChoice: {e:?}")) } /// Returns a read-lock to core `ProtoArray` struct. diff --git a/consensus/proto_array/src/ssz_container.rs b/consensus/proto_array/src/ssz_container.rs index 63f75ed0a2f..1a20ef967ad 100644 --- a/consensus/proto_array/src/ssz_container.rs +++ b/consensus/proto_array/src/ssz_container.rs @@ -2,10 +2,12 @@ use crate::proto_array::ProposerBoost; use crate::{ proto_array::{CountUnrealizedFull, ProtoArray, ProtoNode}, proto_array_fork_choice::{ElasticList, ProtoArrayForkChoice, VoteTracker}, + Error, JustifiedBalances, }; use ssz::{four_byte_option_impl, Encode}; use ssz_derive::{Decode, Encode}; use std::collections::HashMap; +use std::convert::TryFrom; use types::{Checkpoint, Hash256}; // Define a "legacy" implementation of `Option` which uses four bytes for encoding the union @@ -30,7 +32,7 @@ impl From<&ProtoArrayForkChoice> for SszContainer { Self { votes: from.votes.0.clone(), - balances: from.balances.clone(), + balances: from.balances.effective_balances.clone(), prune_threshold: proto_array.prune_threshold, justified_checkpoint: proto_array.justified_checkpoint, finalized_checkpoint: proto_array.finalized_checkpoint, @@ -41,8 +43,12 @@ impl From<&ProtoArrayForkChoice> for SszContainer { } } -impl From<(SszContainer, CountUnrealizedFull)> for ProtoArrayForkChoice { - fn from((from, count_unrealized_full): (SszContainer, CountUnrealizedFull)) -> Self { +impl TryFrom<(SszContainer, CountUnrealizedFull)> for ProtoArrayForkChoice { + type Error = Error; + + fn try_from( + (from, count_unrealized_full): (SszContainer, CountUnrealizedFull), + ) -> Result { let proto_array = ProtoArray { prune_threshold: from.prune_threshold, justified_checkpoint: from.justified_checkpoint, @@ -53,10 +59,10 @@ impl From<(SszContainer, CountUnrealizedFull)> for ProtoArrayForkChoice { count_unrealized_full, }; - Self { + Ok(Self { proto_array, votes: ElasticList(from.votes), - balances: from.balances, - } + balances: JustifiedBalances::from_effective_balances(from.balances)?, + }) } } diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 5e3fec3f3d4..60823552917 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1488,7 +1488,7 @@ fn proposer_re_org_fraction() { .flag("enable-proposer-re-orgs", Some("true")) .flag("proposer-re-org-fraction", Some("90")) .run() - .with_config(|config| assert_eq!(config.chain.re_org_threshold, Some(90))); + .with_config(|config| assert_eq!(config.chain.re_org_threshold.unwrap().0, 90)); } #[test] From ab41479b936a3c1046e10679c2b29917e849e5e4 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 10 Oct 2022 16:11:01 +1100 Subject: [PATCH 12/37] Fix participation check --- beacon_node/beacon_chain/src/beacon_chain.rs | 1 + .../http_api/tests/interactive_tests.rs | 12 ++++++++++ consensus/fork_choice/src/fork_choice.rs | 11 +++++++++ .../src/proto_array_fork_choice.rs | 24 +++++++++++++------ 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b1a558692e9..099752029b3 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3340,6 +3340,7 @@ impl BeaconChain { "ffg_competitive" => proposer_head.ffg_competitive, "cache_hit" => cache_hit, "shuffling_stable" => proposer_head.shuffling_stable, + "participation_ok" => proposer_head.participation_ok, ); None } diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 5309c4d32a5..0758495db1d 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -66,6 +66,7 @@ pub async fn proposer_boost_re_org_test( // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing // `validator_count // 32`. let validator_count = 32; + let all_validators = (0..validator_count).collect::>(); let num_initial = head_slot.as_u64() - 1; let tester = InteractiveTester::::new_with_mutator( @@ -101,6 +102,17 @@ pub async fn proposer_boost_re_org_test( let block_a_root = harness.head_block_root(); let state_a = harness.get_current_state(); + // Attest to block A during slot B. + harness.advance_slot(); + let block_a_empty_votes = harness.make_attestations( + &all_validators, + &state_a, + state_a.canonical_root(), + block_a_root.into(), + slot_b, + ); + harness.process_attestations(block_a_empty_votes); + // Produce block B and process it halfway through the slot. let (block_b, mut state_b) = harness.make_block(state_a.clone(), slot_b).await; let block_b_root = block_b.canonical_root(); diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 6ac1bcb8707..ff305e9023a 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -64,6 +64,9 @@ pub enum Error { current_slot: Slot, fc_store_slot: Slot, }, + ProposerBoostNotExpiredForGetProposerHead { + proposer_boost_root: Hash256, + }, UnrealizedVoteProcessing(state_processing::EpochProcessingError), ParticipationCacheBuild(BeaconStateError), ValidatorStatuses(BeaconStateError), @@ -577,6 +580,14 @@ where }); } + // Similarly, the proposer boost for the previous head should already have expired. + let proposer_boost_root = self.fc_store.proposer_boost_root(); + if !proposer_boost_root.is_zero() { + return Err(Error::ProposerBoostNotExpiredForGetProposerHead { + proposer_boost_root, + }); + } + self.proto_array .get_proposer_head::( current_slot, diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index 25e857cae4b..1f53e759e65 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -394,17 +394,20 @@ impl ProtoArrayForkChoice { // To prevent excessive re-orgs when the chain is struggling, only re-org when participation // is above the configured threshold. This should not overflow. - let participation_ok = parent_node.weight - >= justified_balances.total_effective_balance * participation_threshold.0 / 100; + let participation_committee_threshold = + calculate_proposer_boost::(justified_balances, participation_threshold.0) + .ok_or_else(|| { + "overflow calculating committee weight for participation threshold".to_string() + })?; + let participation_ok = + parent_node.weight >= participation_committee_threshold.saturating_mul(2); // Only re-org if the head's weight is less than the configured committee fraction. let re_org_weight_threshold = calculate_proposer_boost::(justified_balances, re_org_threshold.0).ok_or_else( - || "overflow calculating committee weight for proposer boost".to_string(), + || "overflow calculating committee weight for re-org threshold".to_string(), )?; - let canonical_head_weight = self - .get_block_unique_weight(canonical_head, &justified_balances.effective_balances) - .map_err(|e| format!("overflow calculating head weight: {:?}", e))?; + let canonical_head_weight = head_node.weight; let is_weak_head = canonical_head_weight < re_org_weight_threshold; let re_org_head = (is_single_slot_re_org @@ -430,13 +433,20 @@ impl ProtoArrayForkChoice { /// This weight is the weight unique to the block, *not* including the weight of its ancestors. /// /// Any `proposer_boost` in effect is ignored: only attestations are counted. - fn get_block_unique_weight( + // FIXME(sproul): consider deleting + pub fn get_block_unique_weight( &self, block_root: Hash256, justified_balances: &[u64], + equivocating_indices: &BTreeSet, ) -> Result { let mut unique_weight = 0u64; for (validator_index, vote) in self.votes.iter().enumerate() { + // Skip equivocating validators. + if equivocating_indices.contains(&(validator_index as u64)) { + continue; + } + // Check the `next_root` as we care about the most recent attestations, including ones // from the previous slot that have just been dequeued but haven't run fully through // fork choice yet. From 7d6364a88b992eba2880274f5a92c11b5a6ecc73 Mon Sep 17 00:00:00 2001 From: pawan Date: Mon, 18 Oct 2021 15:05:59 -0700 Subject: [PATCH 13/37] Allow forking in execution block generator --- beacon_node/execution_layer/src/lib.rs | 13 ++ .../test_utils/execution_block_generator.rs | 132 ++++++++++++++---- .../src/test_utils/mock_execution_layer.rs | 15 ++ .../execution_layer/src/test_utils/mod.rs | 2 +- 4 files changed, 137 insertions(+), 25 deletions(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index db1864eba0a..835d2f6356e 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1436,6 +1436,19 @@ mod test { .await; } + #[tokio::test] + async fn test_forked_terminal_block() { + let (mock, block_hash) = MockExecutionLayer::default_params() + .move_to_terminal_block() + .produce_forked_pow_block(); + assert!(mock + .el + .is_valid_terminal_pow_block_hash(block_hash) + .await + .unwrap() + .unwrap()); + } + #[tokio::test] async fn finds_valid_terminal_block_hash() { let runtime = TestRuntime::default(); diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 3620a02dfbb..7cc2fd8883f 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -110,8 +110,10 @@ pub struct ExecutionBlockGenerator { /* * Common database */ + head_block: Option>, + finalized_block_hash: Option, blocks: HashMap>, - block_hashes: HashMap, + block_hashes: HashMap>, /* * PoW block parameters */ @@ -133,6 +135,8 @@ impl ExecutionBlockGenerator { terminal_block_hash: ExecutionBlockHash, ) -> Self { let mut gen = Self { + head_block: <_>::default(), + finalized_block_hash: <_>::default(), blocks: <_>::default(), block_hashes: <_>::default(), terminal_total_difficulty, @@ -149,13 +153,7 @@ impl ExecutionBlockGenerator { } pub fn latest_block(&self) -> Option> { - let hash = *self - .block_hashes - .iter() - .max_by_key(|(number, _)| *number) - .map(|(_, hash)| hash)?; - - self.block_by_hash(hash) + self.head_block.clone() } pub fn latest_execution_block(&self) -> Option { @@ -164,8 +162,18 @@ impl ExecutionBlockGenerator { } pub fn block_by_number(&self, number: u64) -> Option> { - let hash = *self.block_hashes.get(&number)?; - self.block_by_hash(hash) + // Get the latest canonical head block + let mut latest_block = self.latest_block()?; + loop { + let block_number = latest_block.block_number(); + if block_number < number { + return None; + } + if block_number == number { + return Some(latest_block); + } + latest_block = self.block_by_hash(latest_block.parent_hash())?; + } } pub fn execution_block_by_number(&self, number: u64) -> Option { @@ -226,10 +234,16 @@ impl ExecutionBlockGenerator { } pub fn insert_pow_block(&mut self, block_number: u64) -> Result<(), String> { + if let Some(finalized_block_hash) = self.finalized_block_hash { + return Err(format!( + "terminal block {} has been finalized. PoW chain has stopped building", + finalized_block_hash + )); + } let parent_hash = if block_number == 0 { ExecutionBlockHash::zero() - } else if let Some(hash) = self.block_hashes.get(&(block_number - 1)) { - *hash + } else if let Some(block) = self.block_by_number(block_number - 1) { + block.block_hash() } else { return Err(format!( "parent with block number {} not found", @@ -244,33 +258,91 @@ impl ExecutionBlockGenerator { parent_hash, )?; - self.insert_block(Block::PoW(block)) + // Insert block into block tree + let _ = self.insert_block(Block::PoW(block.clone()))?; + + // Set head + if let Some(head_total_difficulty) = + self.head_block.as_ref().and_then(|b| b.total_difficulty()) + { + if block.total_difficulty >= head_total_difficulty { + self.head_block = Some(Block::PoW(block)); + } + } else { + self.head_block = Some(Block::PoW(block)); + } + Ok(()) + } + + /// Insert a PoW block given the parent hash. + /// + /// Returns `Ok(hash)` of the inserted block. + /// Returns an error if the `parent_hash` does not exist in the block tree or + /// if the parent block is the terminal block. + pub fn insert_pow_block_by_hash( + &mut self, + parent_hash: ExecutionBlockHash, + ) -> Result { + let parent_block = self.block_by_hash(parent_hash).ok_or_else(|| { + format!( + "Block corresponding to parent hash does not exist: {}", + parent_hash + ) + })?; + let block = generate_pow_block( + self.terminal_total_difficulty, + self.terminal_block_number, + parent_block.block_number() + 1, + parent_hash, + )?; + + let hash = self.insert_block(Block::PoW(block.clone()))?; + // Set head + if let Some(head_total_difficulty) = + self.head_block.as_ref().and_then(|b| b.total_difficulty()) + { + if block.total_difficulty >= head_total_difficulty { + self.head_block = Some(Block::PoW(block)); + } + } else { + self.head_block = Some(Block::PoW(block)); + } + Ok(hash) } - pub fn insert_block(&mut self, block: Block) -> Result<(), String> { + pub fn insert_block(&mut self, block: Block) -> Result { if self.blocks.contains_key(&block.block_hash()) { return Err(format!("{:?} is already known", block.block_hash())); - } else if self.block_hashes.contains_key(&block.block_number()) { - return Err(format!( - "block {} is already known, forking is not supported", - block.block_number() - )); - } else if block.block_number() != 0 && !self.blocks.contains_key(&block.parent_hash()) { + } else if block.parent_hash() != ExecutionBlockHash::zero() + && !self.blocks.contains_key(&block.parent_hash()) + { return Err(format!("parent block {:?} is unknown", block.parent_hash())); + } else if let Some(hashes) = self.block_hashes.get_mut(&block.block_number()) { + hashes.push(block.block_hash()); + } else { + self.block_hashes + .insert(block.block_number(), vec![block.block_hash()]); } self.insert_block_without_checks(block) } - pub fn insert_block_without_checks(&mut self, block: Block) -> Result<(), String> { + pub fn insert_block_without_checks( + &mut self, + block: Block, + ) -> Result { + let block_hash = block.block_hash(); self.block_hashes - .insert(block.block_number(), block.block_hash()); - self.blocks.insert(block.block_hash(), block); + .entry(block.block_number()) + .or_insert_with(Vec::new) + .push(block_hash); + self.blocks.insert(block_hash, block); - Ok(()) + Ok(block_hash) } pub fn modify_last_block(&mut self, block_modifier: impl FnOnce(&mut Block)) { + /* FIXME(sproul): fix this if let Some((last_block_hash, block_number)) = self.block_hashes.keys().max().and_then(|block_number| { self.block_hashes @@ -288,6 +360,7 @@ impl ExecutionBlockGenerator { self.block_hashes.insert(block_number, block.block_hash()); self.blocks.insert(block.block_hash(), block); } + */ } pub fn get_payload(&mut self, id: &PayloadId) -> Option> { @@ -405,6 +478,17 @@ impl ExecutionBlockGenerator { } }; + self.head_block = Some( + self.blocks + .get(&forkchoice_state.head_block_hash) + .unwrap() + .clone(), + ); + + if forkchoice_state.finalized_block_hash != ExecutionBlockHash::zero() { + self.finalized_block_hash = Some(forkchoice_state.finalized_block_hash); + } + Ok(JsonForkchoiceUpdatedV1Response { payload_status: JsonPayloadStatusV1 { status: JsonPayloadStatusV1Status::Valid, diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 065abc93609..be24b3b14d4 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -234,6 +234,21 @@ impl MockExecutionLayer { self } + pub fn produce_forked_pow_block(self) -> (Self, ExecutionBlockHash) { + let head_block = self + .server + .execution_block_generator() + .latest_block() + .unwrap(); + + let block_hash = self + .server + .execution_block_generator() + .insert_pow_block_by_hash(head_block.parent_hash()) + .unwrap(); + (self, block_hash) + } + pub async fn with_terminal_block<'a, U, V>(self, func: U) -> Self where U: Fn(ChainSpec, ExecutionLayer, Option) -> V, diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index aaeea8aa5af..a8b8ea58ca8 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -357,7 +357,7 @@ impl MockServer { // The EF tests supply blocks out of order, so we must import them "without checks" and // trust they form valid chains. .insert_block_without_checks(block) - .unwrap() + .unwrap(); } pub fn get_block(&self, block_hash: ExecutionBlockHash) -> Option> { From 94df5a1fdf4ffb4f88be70fb3f26de448c56f3d4 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 11 Oct 2022 12:23:01 +1100 Subject: [PATCH 14/37] Fix and test fcU timing --- beacon_node/beacon_chain/src/beacon_chain.rs | 388 +++++++++++------- .../beacon_chain/src/canonical_head.rs | 29 +- .../src/engine_api/json_structures.rs | 12 +- beacon_node/execution_layer/src/lib.rs | 9 + .../src/test_utils/handle_rpc.rs | 8 + .../execution_layer/src/test_utils/hook.rs | 34 ++ .../execution_layer/src/test_utils/mod.rs | 4 + beacon_node/http_api/tests/common.rs | 1 + .../http_api/tests/interactive_tests.rs | 166 +++++++- 9 files changed, 480 insertions(+), 171 deletions(-) create mode 100644 beacon_node/execution_layer/src/test_utils/hook.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 099752029b3..5817eff437e 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -49,9 +49,7 @@ use crate::validator_monitor::{ HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS, }; use crate::validator_pubkey_cache::ValidatorPubkeyCache; -use crate::BeaconForkChoiceStore; -use crate::BeaconSnapshot; -use crate::{metrics, BeaconChainError}; +use crate::{metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, CachedHead}; use eth2::types::{EventKind, SseBlock, SyncDuty}; use execution_layer::{ BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, PayloadAttributes, PayloadStatus, @@ -184,6 +182,17 @@ pub enum ProduceBlockVerification { NoVerification, } +pub struct PrePayloadAttributes { + pub proposer_index: u64, + pub prev_randao: Hash256, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OverrideForkchoiceUpdate { + Yes, + AlreadyApplied, +} + /// The accepted clock drift for nodes gossiping blocks and attestations. See: /// /// https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/p2p-interface.md#configuration @@ -3345,7 +3354,110 @@ impl BeaconChain { None } - /// Determine whether a fork choice update to the execution layer should be suppressed. + /// + pub fn get_pre_payload_attributes( + &self, + proposal_slot: Slot, + proposer_head: Hash256, + cached_head: &CachedHead, + ) -> Result, Error> { + let proposal_epoch = proposal_slot.epoch(T::EthSpec::slots_per_epoch()); + + let head_block_root = cached_head.head_block_root(); + let parent_block_root = cached_head.parent_block_root(); + + // The proposer head must be equal to the canonical head or its parent. + if proposer_head != head_block_root && proposer_head != parent_block_root { + warn!( + self.log, + "Unable to compute payload attributes"; + "block_root" => ?proposer_head, + "head_block_root" => ?head_block_root, + ); + return Ok(None); + } + + // Compute the proposer index. + let head_epoch = cached_head.head_slot().epoch(T::EthSpec::slots_per_epoch()); + let shuffling_decision_root = if head_epoch == proposal_epoch { + cached_head + .snapshot + .beacon_state + .proposer_shuffling_decision_root(proposer_head)? + } else { + proposer_head + }; + let cached_proposer = self + .beacon_proposer_cache + .lock() + .get_slot::(shuffling_decision_root, proposal_slot); + let proposer_index = if let Some(proposer) = cached_proposer { + proposer.index as u64 + } else { + if head_epoch + 2 < proposal_epoch { + warn!( + self.log, + "Skipping proposer preparation"; + "msg" => "this is a non-critical issue that can happen on unhealthy nodes or \ + networks.", + "proposal_epoch" => proposal_epoch, + "head_epoch" => head_epoch, + ); + + // Don't skip the head forward more than two epochs. This avoids burdening an + // unhealthy node. + // + // Although this node might miss out on preparing for a proposal, they should still + // be able to propose. This will prioritise beacon chain health over efficient + // packing of execution blocks. + return Ok(None); + } + + let (proposers, decision_root, _, fork) = + compute_proposer_duties_from_head(proposal_epoch, self)?; + + let proposer_offset = (proposal_slot % T::EthSpec::slots_per_epoch()).as_usize(); + let proposer = *proposers + .get(proposer_offset) + .ok_or(BeaconChainError::NoProposerForSlot(proposal_slot))?; + + self.beacon_proposer_cache.lock().insert( + proposal_epoch, + decision_root, + proposers, + fork, + )?; + + // It's possible that the head changes whilst computing these duties. If so, abandon + // this routine since the change of head would have also spawned another instance of + // this routine. + // + // Exit now, after updating the cache. + if decision_root != shuffling_decision_root { + warn!( + self.log, + "Head changed during proposer preparation"; + ); + return Ok(None); + } + + proposer as u64 + }; + + // Get the `prev_randao` value. + let prev_randao = if proposer_head == parent_block_root { + cached_head.parent_random() + } else { + cached_head.head_random() + }?; + + Ok(Some(PrePayloadAttributes { + proposer_index, + prev_randao, + })) + } + + /// Determine whether a fork choice update to the execution layer should be overridden. /// /// This is *only* necessary when proposer re-orgs are enabled, because we have to prevent the /// execution layer from enshrining the block we want to re-org as the head. @@ -3353,16 +3465,18 @@ impl BeaconChain { /// This function uses heuristics that align quite closely but not exactly with the re-org /// conditions set out in `get_state_for_re_org` and `get_proposer_head`. The differences are /// documented below. - async fn should_suppress_fork_choice_update( + fn overridden_forkchoice_update_params( &self, + canonical_forkchoice_params: ForkchoiceUpdateParameters, current_slot: Slot, - head_block_root: Hash256, - ) -> Result { - // Never supress if proposer re-orgs are disabled. + ) -> Result { + // Never override if proposer re-orgs are disabled. if self.config.re_org_threshold.is_none() { - return Ok(false); + return Ok(canonical_forkchoice_params); } + let head_block_root = canonical_forkchoice_params.head_root; + // Load details of the head block and its parent from fork choice. let (head_node, parent_node) = { let mut nodes = self @@ -3376,7 +3490,7 @@ impl BeaconChain { .collect::>(); if nodes.len() != 2 { - return Ok(false); + return Ok(canonical_forkchoice_params); } let parent = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; @@ -3392,6 +3506,7 @@ impl BeaconChain { // If our proposal fails entirely we will attest to the wrong head during // `re_org_block_slot` and only re-align with the canonical chain 500ms before the start of // the next slot (i.e. `head_slot + 2`). + // FIXME(sproul): add some timing conditions here let current_slot_ok = head_node.slot == current_slot || re_org_block_slot == current_slot; // Only attempt single slot re-orgs, and not at epoch boundaries. @@ -3404,14 +3519,7 @@ impl BeaconChain { && parent_node.unrealized_finalized_checkpoint == head_node.unrealized_finalized_checkpoint; - // Check that this node has a proposer prepared to execute a re-org. - let prepared_for_re_org = self - .execution_layer - .as_ref() - .ok_or(Error::ExecutionLayerMissing)? - .payload_attributes(re_org_block_slot, parent_node.root) - .await - .is_some(); + // FIXME(sproul): add participation check // Check that the head block arrived late and is vulnerable to a re-org. This check is only // a heuristic compared to the proper weight check in `get_state_for_re_org`, the reason @@ -3421,13 +3529,40 @@ impl BeaconChain { let head_block_late = self.block_observed_after_attestation_deadline(head_block_root, head_node.slot); - let might_re_org = current_slot_ok - && block_slot_ok - && ffg_competitive - && prepared_for_re_org - && head_block_late; + let might_re_org = current_slot_ok && block_slot_ok && ffg_competitive && head_block_late; - Ok(might_re_org) + if !might_re_org { + debug!( + self.log, + "Not overriding fork choice update"; + "current_slot_ok" => current_slot_ok, + "block_slot_ok" => block_slot_ok, + "ffg_competitive" => ffg_competitive, + "head_block_late" => head_block_late + ); + return Ok(canonical_forkchoice_params); + } + + let parent_head_hash = if let Some(hash) = parent_node.execution_status.block_hash() { + hash + } else { + warn!( + self.log, + "Missing exec block hash for parent"; + "parent_root" => ?parent_node.root, + "status" => ?parent_node.execution_status, + ); + return Ok(canonical_forkchoice_params); + }; + + let forkchoice_update_params = ForkchoiceUpdateParameters { + head_root: parent_node.root, + head_hash: Some(parent_head_hash), + justified_hash: canonical_forkchoice_params.justified_hash, + finalized_hash: canonical_forkchoice_params.finalized_hash, + }; + + Ok(forkchoice_update_params) } /// Check if the block with `block_root` was observed after the attestation deadline of `slot`. @@ -4012,7 +4147,6 @@ impl BeaconChain { current_slot: Slot, ) -> Result<(), Error> { let prepare_slot = current_slot + 1; - let prepare_epoch = prepare_slot.epoch(T::EthSpec::slots_per_epoch()); // There's no need to run the proposer preparation routine before the bellatrix fork. if self.slot_is_prior_to_bellatrix(prepare_slot) { @@ -4030,156 +4164,96 @@ impl BeaconChain { return Ok(()); } - // Atomically read some values from the canonical head, whilst avoiding holding the cached - // head `Arc` any longer than necessary. + // Load the cached head and its forkchoice update parameters. // // Use a blocking task since blocking the core executor on the canonical head read lock can // block the core tokio executor. let chain = self.clone(); - let (head_slot, head_root, head_decision_root, head_random, forkchoice_update_params) = - self.spawn_blocking_handle( + let maybe_prep_data = self + .spawn_blocking_handle( move || { let cached_head = chain.canonical_head.cached_head(); - let head_block_root = cached_head.head_block_root(); - let decision_root = cached_head - .snapshot - .beacon_state - .proposer_shuffling_decision_root(head_block_root)?; - Ok::<_, Error>(( - cached_head.head_slot(), - head_block_root, - decision_root, - cached_head.head_random()?, - cached_head.forkchoice_update_parameters(), - )) + + // Don't bother with proposer prep if the head is more than + // `PREPARE_PROPOSER_HISTORIC_EPOCHS` prior to the current slot. + // + // This prevents the routine from running during sync. + let head_slot = cached_head.head_slot(); + if head_slot + T::EthSpec::slots_per_epoch() * PREPARE_PROPOSER_HISTORIC_EPOCHS + < current_slot + { + debug!( + chain.log, + "Head too old for proposer prep"; + "head_slot" => head_slot, + "current_slot" => current_slot, + ); + return Ok(None); + } + + let canonical_fcu_params = cached_head.forkchoice_update_parameters(); + let fcu_params = chain + .overridden_forkchoice_update_params(canonical_fcu_params, current_slot)?; + let pre_payload_attributes = chain.get_pre_payload_attributes( + prepare_slot, + fcu_params.head_root, + &cached_head, + )?; + Ok::<_, Error>(Some((fcu_params, pre_payload_attributes))) }, - "prepare_beacon_proposer_fork_choice_read", + "prepare_beacon_proposer_head_read", ) .await??; - let head_epoch = head_slot.epoch(T::EthSpec::slots_per_epoch()); - - // Don't bother with proposer prep if the head is more than - // `PREPARE_PROPOSER_HISTORIC_EPOCHS` prior to the current slot. - // - // This prevents the routine from running during sync. - if head_slot + T::EthSpec::slots_per_epoch() * PREPARE_PROPOSER_HISTORIC_EPOCHS - < current_slot - { - debug!( - self.log, - "Head too old for proposer prep"; - "head_slot" => head_slot, - "current_slot" => current_slot, - ); - return Ok(()); - } - - // Ensure that the shuffling decision root is correct relative to the epoch we wish to - // query. - let shuffling_decision_root = if head_epoch == prepare_epoch { - head_decision_root - } else { - head_root - }; - - // Read the proposer from the proposer cache. - let cached_proposer = self - .beacon_proposer_cache - .lock() - .get_slot::(shuffling_decision_root, prepare_slot); - let proposer = if let Some(proposer) = cached_proposer { - proposer.index - } else { - if head_epoch + 2 < prepare_epoch { - warn!( - self.log, - "Skipping proposer preparation"; - "msg" => "this is a non-critical issue that can happen on unhealthy nodes or \ - networks.", - "prepare_epoch" => prepare_epoch, - "head_epoch" => head_epoch, - ); - - // Don't skip the head forward more than two epochs. This avoids burdening an - // unhealthy node. - // - // Although this node might miss out on preparing for a proposal, they should still - // be able to propose. This will prioritise beacon chain health over efficient - // packing of execution blocks. - return Ok(()); - } - - let (proposers, decision_root, _, fork) = - compute_proposer_duties_from_head(prepare_epoch, self)?; - - let proposer_index = prepare_slot.as_usize() % (T::EthSpec::slots_per_epoch() as usize); - let proposer = *proposers - .get(proposer_index) - .ok_or(BeaconChainError::NoProposerForSlot(prepare_slot))?; - self.beacon_proposer_cache.lock().insert( - prepare_epoch, - decision_root, - proposers, - fork, - )?; - - // It's possible that the head changes whilst computing these duties. If so, abandon - // this routine since the change of head would have also spawned another instance of - // this routine. - // - // Exit now, after updating the cache. - if decision_root != shuffling_decision_root { - warn!( - self.log, - "Head changed during proposer preparation"; - ); + let (forkchoice_update_params, pre_payload_attributes) = + if let Some((fcu, Some(pre_payload))) = maybe_prep_data { + (fcu, pre_payload) + } else { + // Appropriate log messages have already been logged above and in + // `get_pre_payload_attributes`. return Ok(()); - } - - proposer - }; + }; // If the execution layer doesn't have any proposer data for this validator then we assume // it's not connected to this BN and no action is required. + let proposer = pre_payload_attributes.proposer_index; if !execution_layer - .has_proposer_preparation_data(proposer as u64) + .has_proposer_preparation_data(proposer) .await { return Ok(()); } + let head_root = forkchoice_update_params.head_root; let payload_attributes = PayloadAttributes { timestamp: self .slot_clock .start_of(prepare_slot) .ok_or(Error::InvalidSlot(prepare_slot))? .as_secs(), - prev_randao: head_random, - suggested_fee_recipient: execution_layer - .get_suggested_fee_recipient(proposer as u64) - .await, + prev_randao: pre_payload_attributes.prev_randao, + suggested_fee_recipient: execution_layer.get_suggested_fee_recipient(proposer).await, }; debug!( self.log, "Preparing beacon proposer"; "payload_attributes" => ?payload_attributes, - "head_root" => ?head_root, "prepare_slot" => prepare_slot, "validator" => proposer, + "parent_root" => ?head_root, ); let already_known = execution_layer - .insert_proposer(prepare_slot, head_root, proposer as u64, payload_attributes) + .insert_proposer(prepare_slot, head_root, proposer, payload_attributes) .await; + // Only push a log to the user if this is the first time we've seen this proposer for this // slot. if !already_known { info!( self.log, "Prepared beacon proposer"; - "already_known" => already_known, "prepare_slot" => prepare_slot, "validator" => proposer, "parent_root" => ?head_root, @@ -4204,27 +4278,24 @@ impl BeaconChain { return Ok(()); }; - // If either of the following are true, send a fork-choice update message to the - // EL: - // - // 1. We're in the tail-end of the slot (as defined by - // PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR) - // 2. The head block is one slot (or less) behind the prepare slot (e.g., we're - // preparing for the next slot and the block at the current slot is already - // known). + // If we are close enough to the proposal slot, send an fcU, which will have payload + // attributes filled in by the execution layer cache we just primed. if till_prepare_slot <= self.slot_clock.slot_duration() / PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR - || head_slot + 1 >= prepare_slot { debug!( self.log, - "Pushing update to prepare proposer"; + "Sending forkchoiceUpdate for proposer prep"; "till_prepare_slot" => ?till_prepare_slot, "prepare_slot" => prepare_slot ); - self.update_execution_engine_forkchoice(current_slot, forkchoice_update_params) - .await?; + self.update_execution_engine_forkchoice( + current_slot, + forkchoice_update_params, + OverrideForkchoiceUpdate::AlreadyApplied, + ) + .await?; } Ok(()) @@ -4233,7 +4304,8 @@ impl BeaconChain { pub async fn update_execution_engine_forkchoice( self: &Arc, current_slot: Slot, - params: ForkchoiceUpdateParameters, + input_params: ForkchoiceUpdateParameters, + override_forkchoice_update: OverrideForkchoiceUpdate, ) -> Result<(), Error> { let next_slot = current_slot + 1; @@ -4255,6 +4327,19 @@ impl BeaconChain { .as_ref() .ok_or(Error::ExecutionLayerMissing)?; + // Determine whether to override the forkchoiceUpdated message if we want to re-org + // the current head at the next slot. + let params = if override_forkchoice_update == OverrideForkchoiceUpdate::Yes { + let chain = self.clone(); + self.spawn_blocking_handle( + move || chain.overridden_forkchoice_update_params(input_params, current_slot), + "update_execution_engine_forkchoice_override", + ) + .await?? + } else { + input_params + }; + // Take the global lock for updating the execution engine fork choice. // // Whilst holding this lock we must: @@ -4329,21 +4414,6 @@ impl BeaconChain { } }; - // Determine whether to suppress the forkchoiceUpdated message if we want to re-org - // the current head at the next slot. - if self - .should_suppress_fork_choice_update(current_slot, head_block_root) - .await? - { - debug!( - self.log, - "Suppressing fork choice update"; - "head_block_root" => ?head_block_root, - "current_slot" => current_slot, - ); - return Ok(()); - } - let forkchoice_updated_response = execution_layer .notify_forkchoice_updated( head_hash, diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index c9bd6db0e67..fd221b4f845 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -34,7 +34,8 @@ use crate::persisted_fork_choice::PersistedForkChoice; use crate::{ beacon_chain::{ - BeaconForkChoice, BeaconStore, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT, FORK_CHOICE_DB_KEY, + BeaconForkChoice, BeaconStore, OverrideForkchoiceUpdate, + BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT, FORK_CHOICE_DB_KEY, }, block_times_cache::BlockTimesCache, events::ServerSentEventHandler, @@ -114,6 +115,11 @@ impl CachedHead { self.snapshot.beacon_block_root } + /// Returns the root of the parent of the head block. + pub fn parent_block_root(&self) -> Hash256 { + self.snapshot.beacon_block.parent_root() + } + /// Returns root of the `BeaconState` at the head of the beacon chain. /// /// ## Note @@ -146,6 +152,21 @@ impl CachedHead { Ok(root) } + /// Returns the randao mix for the parent of the block at the head of the chain. + /// + /// This is useful for re-orging the current head. The parent's RANDAO value is read from + /// the head's execution payload because it is unavailable in the beacon state's RANDAO mixes + /// array after being overwritten by the head block's RANDAO mix. + /// + /// This will error if the head block is not execution-enabled (post Bellatrix). + pub fn parent_random(&self) -> Result { + self.snapshot + .beacon_block + .message() + .execution_payload() + .map(|payload| payload.prev_randao()) + } + /// Returns the active validator count for the current epoch of the head state. /// /// Should only return `None` if the caches have not been built on the head state (this should @@ -1124,7 +1145,11 @@ fn spawn_execution_layer_updates( } if let Err(e) = chain - .update_execution_engine_forkchoice(current_slot, forkchoice_update_params) + .update_execution_engine_forkchoice( + current_slot, + forkchoice_update_params, + OverrideForkchoiceUpdate::Yes, + ) .await { crit!( diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 31aa79f0559..677533fb7a1 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -28,7 +28,7 @@ pub struct JsonResponseBody { pub id: serde_json::Value, } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(transparent)] pub struct TransparentJsonPayloadId(#[serde(with = "eth2_serde_utils::bytes_8_hex")] pub PayloadId); @@ -226,7 +226,7 @@ impl From> for ExecutionPayload { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct JsonPayloadAttributesV1 { #[serde(with = "eth2_serde_utils::u64_hex_be")] @@ -269,7 +269,7 @@ impl From for PayloadAttributes { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct JsonForkChoiceStateV1 { pub head_block_hash: ExecutionBlockHash, @@ -311,7 +311,7 @@ impl From for ForkChoiceState { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum JsonPayloadStatusV1Status { Valid, @@ -321,7 +321,7 @@ pub enum JsonPayloadStatusV1Status { InvalidBlockHash, } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct JsonPayloadStatusV1 { pub status: JsonPayloadStatusV1Status, @@ -386,7 +386,7 @@ impl From for PayloadStatusV1 { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct JsonForkchoiceUpdatedV1Response { pub payload_status: JsonPayloadStatusV1, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 835d2f6356e..29e00e47af9 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1025,6 +1025,7 @@ impl ExecutionLayer { "finalized_block_hash" => ?finalized_block_hash, "justified_block_hash" => ?justified_block_hash, "head_block_hash" => ?head_block_hash, + "head_block_root" => ?head_block_root, "current_slot" => current_slot, ); @@ -1033,6 +1034,12 @@ impl ExecutionLayer { // Compute the "lookahead", the time between when the payload will be produced and now. if let Some(payload_attributes) = payload_attributes { + debug!( + self.log(), + "Sending payload attributes"; + "timestamp" => payload_attributes.timestamp, + "prev_randao" => ?payload_attributes.prev_randao, + ); if let Ok(now) = SystemTime::now().duration_since(UNIX_EPOCH) { let timestamp = Duration::from_secs(payload_attributes.timestamp); if let Some(lookahead) = timestamp.checked_sub(now) { @@ -1041,12 +1048,14 @@ impl ExecutionLayer { lookahead, ); } else { + /* FIXME(sproul): re-enable this, or fix it debug!( self.log(), "Late payload attributes"; "timestamp" => ?timestamp, "now" => ?now, ) + */ } } } diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 975f09fa5e0..d9b95da5fae 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -118,6 +118,14 @@ pub async fn handle_rpc( let forkchoice_state: JsonForkChoiceStateV1 = get_param(params, 0)?; let payload_attributes: Option = get_param(params, 1)?; + if let Some(hook_response) = ctx + .hook + .lock() + .on_forkchoice_updated(forkchoice_state.clone(), payload_attributes.clone()) + { + return Ok(serde_json::to_value(hook_response).unwrap()); + } + let head_block_hash = forkchoice_state.head_block_hash; let mut response = ctx diff --git a/beacon_node/execution_layer/src/test_utils/hook.rs b/beacon_node/execution_layer/src/test_utils/hook.rs new file mode 100644 index 00000000000..fd82f001d3d --- /dev/null +++ b/beacon_node/execution_layer/src/test_utils/hook.rs @@ -0,0 +1,34 @@ +use crate::json_structures::*; + +type ForkChoiceUpdatedHook = dyn Fn( + JsonForkChoiceStateV1, + Option, + ) -> Option + + Send + + Sync; + +pub struct Hook { + forkchoice_updated: Option>, +} + +impl Default for Hook { + fn default() -> Self { + Hook { + forkchoice_updated: None, + } + } +} + +impl Hook { + pub fn on_forkchoice_updated( + &self, + state: JsonForkChoiceStateV1, + payload_attributes: Option, + ) -> Option { + (self.forkchoice_updated.as_ref()?)(state, payload_attributes) + } + + pub fn set_forkchoice_updated_hook(&mut self, f: Box) { + self.forkchoice_updated = Some(f); + } +} diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index a8b8ea58ca8..41476e0f598 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -22,6 +22,7 @@ use types::{EthSpec, ExecutionBlockHash, Uint256}; use warp::{http::StatusCode, Filter, Rejection}; pub use execution_block_generator::{generate_pow_block, Block, ExecutionBlockGenerator}; +pub use hook::Hook; pub use mock_builder::{Context as MockBuilderContext, MockBuilder, Operation, TestingBuilder}; pub use mock_execution_layer::MockExecutionLayer; @@ -32,6 +33,7 @@ pub const DEFAULT_BUILDER_THRESHOLD_WEI: u128 = 1_000_000_000_000_000_000; mod execution_block_generator; mod handle_rpc; +mod hook; mod mock_builder; mod mock_execution_layer; @@ -98,6 +100,7 @@ impl MockServer { static_new_payload_response: <_>::default(), static_forkchoice_updated_response: <_>::default(), static_get_block_by_hash_response: <_>::default(), + hook: <_>::default(), _phantom: PhantomData, }); @@ -419,6 +422,7 @@ pub struct Context { pub static_new_payload_response: Arc>>, pub static_forkchoice_updated_response: Arc>>, pub static_get_block_by_hash_response: Arc>>>, + pub hook: Arc>, pub _phantom: PhantomData, } diff --git a/beacon_node/http_api/tests/common.rs b/beacon_node/http_api/tests/common.rs index 2883ff747c2..8cb61cc28fd 100644 --- a/beacon_node/http_api/tests/common.rs +++ b/beacon_node/http_api/tests/common.rs @@ -64,6 +64,7 @@ impl InteractiveTester { .spec_or_default(spec) .deterministic_keypairs(validator_count) .logger(test_logger()) + .mock_execution_layer() .fresh_ephemeral_store(); if let Some(mutator) = mutator { diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 0758495db1d..14d3fbf4d13 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -1,14 +1,23 @@ //! Generic tests that make use of the (newer) `InteractiveApiTester` use crate::common::*; +use beacon_chain::proposer_prep_service::PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR; use beacon_chain::{ chain_config::ReOrgThreshold, test_utils::{AttestationStrategy, BlockStrategy}, }; use eth2::types::DepositContractData; +use execution_layer::{ForkChoiceState, PayloadAttributes}; +use parking_lot::Mutex; use slot_clock::SlotClock; use state_processing::state_advance::complete_state_advance; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; use tree_hash::TreeHash; -use types::{EthSpec, FullPayload, MainnetEthSpec, Slot}; +use types::{ + Address, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, MainnetEthSpec, + ProposerPreparationData, Slot, +}; type E = MainnetEthSpec; @@ -38,6 +47,53 @@ async fn deposit_contract_custom_network() { assert_eq!(result, expected); } +/// Data structure for tracking fork choice updates received by the mock execution layer. +#[derive(Debug, Default)] +struct ForkChoiceUpdates { + updates: HashMap>, +} + +#[derive(Debug, Clone)] +struct ForkChoiceUpdateMetadata { + received_at: Duration, + state: ForkChoiceState, + payload_attributes: Option, +} + +impl ForkChoiceUpdates { + fn insert(&mut self, update: ForkChoiceUpdateMetadata) { + self.updates + .entry(update.state.head_block_hash) + .or_insert_with(Vec::new) + .push(update); + } + + fn contains_update_for(&self, block_hash: ExecutionBlockHash) -> bool { + self.updates.contains_key(&block_hash) + } + + /// Find the first fork choice update for `head_block_hash` with payload attributes for a + /// block proposal at `proposal_timestamp`. + fn first_update_with_payload_attributes( + &self, + head_block_hash: ExecutionBlockHash, + proposal_timestamp: u64, + ) -> Option { + self.updates + .get(&head_block_hash)? + .iter() + .find(|update| { + update + .payload_attributes + .as_ref() + .map_or(false, |payload_attributes| { + payload_attributes.timestamp == proposal_timestamp + }) + }) + .cloned() + } +} + // Test that the beacon node will try to perform proposer boost re-orgs on late blocks when // configured. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -63,6 +119,10 @@ pub async fn proposer_boost_re_org_test( ) { assert!(head_slot > 0); + // We require a network with execution enabled so we can check EL message timings. + let mut spec = ForkName::Merge.make_genesis_spec(E::default_spec()); + spec.terminal_total_difficulty = 1.into(); + // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing // `validator_count // 32`. let validator_count = 32; @@ -70,7 +130,7 @@ pub async fn proposer_boost_re_org_test( let num_initial = head_slot.as_u64() - 1; let tester = InteractiveTester::::new_with_mutator( - None, + Some(spec), validator_count, Some(Box::new(move |builder| { builder.proposer_re_org_threshold(re_org_threshold.map(ReOrgThreshold)) @@ -78,8 +138,37 @@ pub async fn proposer_boost_re_org_test( ) .await; let harness = &tester.harness; + let mock_el = harness.mock_execution_layer.as_ref().unwrap(); + let execution_ctx = mock_el.server.ctx.clone(); let slot_clock = &harness.chain.slot_clock; + // Move to terminal block. + mock_el.server.all_payloads_valid(); + execution_ctx + .execution_block_generator + .write() + .move_to_terminal_block() + .unwrap(); + + // Send proposer preparation data for all validators. + let proposer_preparation_data = all_validators + .iter() + .map(|i| ProposerPreparationData { + validator_index: *i as u64, + fee_recipient: Address::from_low_u64_be(*i as u64), + }) + .collect::>(); + harness + .chain + .execution_layer + .as_ref() + .unwrap() + .update_proposer_preparation( + head_slot.epoch(E::slots_per_epoch()) + 1, + &proposer_preparation_data, + ) + .await; + // Create some chain depth. harness.advance_slot(); harness @@ -90,6 +179,27 @@ pub async fn proposer_boost_re_org_test( ) .await; + // Start collecting fork choice updates. + let forkchoice_updates = Arc::new(Mutex::new(ForkChoiceUpdates::default())); + let forkchoice_updates_inner = forkchoice_updates.clone(); + let chain_inner = harness.chain.clone(); + + execution_ctx + .hook + .lock() + .set_forkchoice_updated_hook(Box::new(move |state, payload_attributes| { + let received_at = chain_inner.slot_clock.now_duration().unwrap(); + let state = ForkChoiceState::from(state); + let payload_attributes = payload_attributes.map(Into::into); + let update = ForkChoiceUpdateMetadata { + received_at, + state, + payload_attributes, + }; + forkchoice_updates_inner.lock().insert(update); + None + })); + // We set up the following block graph, where B is a block that arrives late and is re-orged // by C. // @@ -100,6 +210,7 @@ pub async fn proposer_boost_re_org_test( let slot_c = slot_a + 2; let block_a_root = harness.head_block_root(); + let block_a = harness.get_block(block_a_root.into()).unwrap(); let state_a = harness.get_current_state(); // Attest to block A during slot B. @@ -126,7 +237,7 @@ pub async fn proposer_boost_re_org_test( None, None, ); - harness.process_block_result(block_b).await.unwrap(); + harness.process_block_result(block_b.clone()).await.unwrap(); // Add attestations to block B. /* FIXME(sproul): implement attestations @@ -140,6 +251,11 @@ pub async fn proposer_boost_re_org_test( ) } */ + // Simulate the scheduled call to prepare proposers at 8 seconds into the slot. + let payload_lookahead = slot_clock.slot_duration() / PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR; + let scheduled_prepare_time = slot_clock.start_of(slot_c).unwrap() - payload_lookahead; + slot_clock.set_current_time(scheduled_prepare_time); + harness.chain.prepare_beacon_proposer(slot_b).await.unwrap(); // Produce block C. while harness.get_current_slot() != slot_c { @@ -173,8 +289,50 @@ pub async fn proposer_boost_re_org_test( } // Applying block C should cause it to become head regardless (re-org or continuation). - let block_root_c = harness.process_block_result(block_c).await.unwrap().into(); + let block_root_c = harness + .process_block_result(block_c.clone()) + .await + .unwrap() + .into(); assert_eq!(harness.head_block_root(), block_root_c); + + // Check the fork choice updates that were sent. + let forkchoice_updates = forkchoice_updates.lock(); + let block_a_exec_hash = block_a.message().execution_payload().unwrap().block_hash(); + let block_b_exec_hash = block_b.message().execution_payload().unwrap().block_hash(); + let block_c_exec_hash = block_c.message().execution_payload().unwrap().block_hash(); + + let block_c_timestamp = block_c.message().execution_payload().unwrap().timestamp(); + + // If we re-orged then no fork choice update for B should have been sent. + assert_eq!( + should_re_org, + !forkchoice_updates.contains_update_for(block_b_exec_hash), + "{block_b_exec_hash:?}" + ); + + // Check the timing of the first fork choice update with payload attributes for block C. + let c_parent_hash = if should_re_org { + block_a_exec_hash + } else { + block_b_exec_hash + }; + let first_update = forkchoice_updates + .first_update_with_payload_attributes(c_parent_hash, block_c_timestamp) + .unwrap(); + let payload_attribs = first_update.payload_attributes.as_ref().unwrap(); + + let lookahead = slot_clock + .start_of(slot_c) + .unwrap() + .checked_sub(first_update.received_at) + .unwrap(); + assert!( + lookahead >= payload_lookahead && lookahead <= 2 * payload_lookahead, + "lookahead={lookahead:?}, timestamp={}, prev_randao={:?}", + payload_attribs.timestamp, + payload_attribs.prev_randao, + ); } // Test that running fork choice before proposing results in selection of the correct head. From e184a4d385be6e677c89771592ac7a95feb1cf9a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 12 Oct 2022 18:39:06 +1100 Subject: [PATCH 15/37] Fix tests, configurable prepare-payload-lookahead --- beacon_node/beacon_chain/src/beacon_chain.rs | 34 +++++++------- beacon_node/beacon_chain/src/chain_config.rs | 11 +++++ beacon_node/beacon_chain/src/lib.rs | 4 +- .../beacon_chain/src/proposer_prep_service.rs | 13 ++---- beacon_node/beacon_chain/src/test_utils.rs | 8 +++- beacon_node/beacon_chain/tests/merge.rs | 2 + .../tests/payload_invalidation.rs | 16 +++++-- .../test_utils/execution_block_generator.rs | 46 ++++++++++--------- .../execution_layer/src/test_utils/mod.rs | 3 +- .../http_api/tests/interactive_tests.rs | 3 +- beacon_node/src/cli.rs | 10 ++++ beacon_node/src/config.rs | 5 ++ lighthouse/tests/beacon_node.rs | 25 ++++++++++ 13 files changed, 120 insertions(+), 60 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5817eff437e..1cc20e9770c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -37,7 +37,6 @@ use crate::observed_operations::{ObservationOutcome, ObservedOperations}; use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_BLOCK_ROOT}; use crate::persisted_fork_choice::PersistedForkChoice; use crate::pre_finalization_cache::PreFinalizationBlockCache; -use crate::proposer_prep_service::PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR; use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache}; use crate::snapshot_cache::{BlockProductionPreState, SnapshotCache}; use crate::sync_committee_verification::{ @@ -182,13 +181,17 @@ pub enum ProduceBlockVerification { NoVerification, } +/// Payload attributes for which the `beacon_chain` crate is responsible. pub struct PrePayloadAttributes { pub proposer_index: u64, pub prev_randao: Hash256, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Define whether a forkchoiceUpdate needs to be checked for an override (`Yes`) or has already +/// been checked (`AlreadyApplied`). It is safe to specify `Yes` even if re-orgs are disabled. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub enum OverrideForkchoiceUpdate { + #[default] Yes, AlreadyApplied, } @@ -3532,14 +3535,6 @@ impl BeaconChain { let might_re_org = current_slot_ok && block_slot_ok && ffg_competitive && head_block_late; if !might_re_org { - debug!( - self.log, - "Not overriding fork choice update"; - "current_slot_ok" => current_slot_ok, - "block_slot_ok" => block_slot_ok, - "ffg_competitive" => ffg_competitive, - "head_block_late" => head_block_late - ); return Ok(canonical_forkchoice_params); } @@ -3555,6 +3550,14 @@ impl BeaconChain { return Ok(canonical_forkchoice_params); }; + debug!( + self.log, + "Fork choice update overridden"; + "canonical_head" => ?head_node.root, + "override" => ?parent_node.root, + "slot" => current_slot, + ); + let forkchoice_update_params = ForkchoiceUpdateParameters { head_root: parent_node.root, head_hash: Some(parent_head_hash), @@ -4137,11 +4140,8 @@ impl BeaconChain { /// The `PayloadAttributes` are used by the EL to give it a look-ahead for preparing an optimal /// set of transactions for a new `ExecutionPayload`. /// - /// This function will result in a call to `forkchoiceUpdated` on the EL if: - /// - /// 1. We're in the tail-end of the slot (as defined by PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR) - /// 2. The head block is one slot (or less) behind the prepare slot (e.g., we're preparing for - /// the next slot and the block at the current slot is already known). + /// This function will result in a call to `forkchoiceUpdated` on the EL if we're in the + /// tail-end of the slot (as defined by `self.config.prepare_payload_lookahead`). pub async fn prepare_beacon_proposer( self: &Arc, current_slot: Slot, @@ -4280,9 +4280,7 @@ impl BeaconChain { // If we are close enough to the proposal slot, send an fcU, which will have payload // attributes filled in by the execution layer cache we just primed. - if till_prepare_slot - <= self.slot_clock.slot_duration() / PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR - { + if till_prepare_slot <= self.config.prepare_payload_lookahead { debug!( self.log, "Sending forkchoiceUpdate for proposer prep"; diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index a1d35eea016..d6e6cf9f5cf 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -1,5 +1,6 @@ pub use proto_array::{CountUnrealizedFull, ParticipationThreshold, ReOrgThreshold}; use serde_derive::{Deserialize, Serialize}; +use std::time::Duration; use types::Checkpoint; pub const DEFAULT_RE_ORG_THRESHOLD: ReOrgThreshold = ReOrgThreshold(10); @@ -7,6 +8,10 @@ pub const DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD: ParticipationThreshold = ParticipationThreshold(80); pub const DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT: u64 = 250; +/// At 12s slot times, the means that the payload preparation routine will run 4s before the start +/// of each slot (`12 / 3 = 4`). +pub const DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR: u32 = 3; + #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] pub struct ChainConfig { /// Maximum number of slots to skip when importing a consensus message (e.g., block, @@ -52,6 +57,11 @@ pub struct ChainConfig { pub paranoid_block_proposal: bool, /// Whether to strictly count unrealized justified votes. pub count_unrealized_full: CountUnrealizedFull, + /// The offset from the start of a proposal slot at which payload attributes should be sent. + /// + /// Low values are useful for execution engines which don't improve their payload after the + /// first call, and high values are useful for ensuring the EL is given ample notice. + pub prepare_payload_lookahead: Duration, } impl Default for ChainConfig { @@ -74,6 +84,7 @@ impl Default for ChainConfig { always_reset_payload_statuses: false, paranoid_block_proposal: false, count_unrealized_full: CountUnrealizedFull::default(), + prepare_payload_lookahead: Duration::from_secs(4), } } } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index fbcd8f7fb76..4abbbda9122 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -45,8 +45,8 @@ pub mod validator_pubkey_cache; pub use self::beacon_chain::{ AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult, - CountUnrealized, ForkChoiceError, ProduceBlockVerification, StateSkipConfig, WhenSlotSkipped, - INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, + CountUnrealized, ForkChoiceError, OverrideForkchoiceUpdate, ProduceBlockVerification, + StateSkipConfig, WhenSlotSkipped, INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, INVALID_JUSTIFIED_PAYLOAD_SHUTDOWN_REASON, MAXIMUM_GOSSIP_CLOCK_DISPARITY, }; pub use self::beacon_snapshot::BeaconSnapshot; diff --git a/beacon_node/beacon_chain/src/proposer_prep_service.rs b/beacon_node/beacon_chain/src/proposer_prep_service.rs index 9cd177b3409..140a9659fce 100644 --- a/beacon_node/beacon_chain/src/proposer_prep_service.rs +++ b/beacon_node/beacon_chain/src/proposer_prep_service.rs @@ -5,13 +5,9 @@ use std::sync::Arc; use task_executor::TaskExecutor; use tokio::time::sleep; -/// At 12s slot times, the means that the payload preparation routine will run 4s before the start -/// of each slot (`12 / 3 = 4`). -pub const PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR: u32 = 3; - /// Spawns a routine which ensures the EL is provided advance notice of any block producers. /// -/// This routine will run once per slot, at `slot_duration / PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR` +/// This routine will run once per slot, at `chain.prepare_payload_lookahead()` /// before the start of each slot. /// /// The service will not be started if there is no `execution_layer` on the `chain`. @@ -38,8 +34,8 @@ async fn proposer_prep_service( loop { match chain.slot_clock.duration_to_next_slot() { Some(duration) => { - let additional_delay = slot_duration - - chain.slot_clock.slot_duration() / PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR; + let additional_delay = + slot_duration.saturating_sub(chain.config.prepare_payload_lookahead); sleep(duration + additional_delay).await; debug!( @@ -65,14 +61,11 @@ async fn proposer_prep_service( }, "proposer_prep_update", ); - - continue; } None => { error!(chain.log, "Failed to read slot clock"); // If we can't read the slot clock, just wait another slot. sleep(slot_duration).await; - continue; } }; } diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index ea51b2ab582..c6c4bd04c02 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -32,7 +32,7 @@ use rand::SeedableRng; use rayon::prelude::*; use sensitive_url::SensitiveUrl; use slog::Logger; -use slot_clock::TestingSlotClock; +use slot_clock::{SlotClock, TestingSlotClock}; use state_processing::per_block_processing::compute_timestamp_at_slot; use state_processing::{ state_advance::{complete_state_advance, partial_state_advance}, @@ -1750,6 +1750,12 @@ where self.chain.slot_clock.advance_slot(); } + /// Advance the clock to `lookahead` before the start of `slot`. + pub fn advance_to_slot_lookahead(&self, slot: Slot, lookahead: Duration) { + let time = self.chain.slot_clock.start_of(slot).unwrap() - lookahead; + self.chain.slot_clock.set_current_time(time); + } + /// Deprecated: Use make_block() instead /// /// Returns a newly created block, signed by the proposer for the given slot. diff --git a/beacon_node/beacon_chain/tests/merge.rs b/beacon_node/beacon_chain/tests/merge.rs index 19e8902a3e8..c8c47c99041 100644 --- a/beacon_node/beacon_chain/tests/merge.rs +++ b/beacon_node/beacon_chain/tests/merge.rs @@ -53,6 +53,7 @@ async fn merge_with_terminal_block_hash_override() { let harness = BeaconChainHarness::builder(E::default()) .spec(spec) + .logger(logging::test_logger()) .deterministic_keypairs(VALIDATOR_COUNT) .fresh_ephemeral_store() .mock_execution_layer() @@ -109,6 +110,7 @@ async fn base_altair_merge_with_terminal_block_after_fork() { let harness = BeaconChainHarness::builder(E::default()) .spec(spec) + .logger(logging::test_logger()) .deterministic_keypairs(VALIDATOR_COUNT) .fresh_ephemeral_store() .mock_execution_layer() diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 2336c3ba994..531fdb63047 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -7,8 +7,8 @@ use beacon_chain::otb_verification_service::{ use beacon_chain::{ canonical_head::{CachedHead, CanonicalHead}, test_utils::{BeaconChainHarness, EphemeralHarnessType}, - BeaconChainError, BlockError, ExecutionPayloadError, StateSkipConfig, WhenSlotSkipped, - INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, + BeaconChainError, BlockError, ExecutionPayloadError, OverrideForkchoiceUpdate, StateSkipConfig, + WhenSlotSkipped, INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON, INVALID_JUSTIFIED_PAYLOAD_SHUTDOWN_REASON, }; use execution_layer::{ @@ -19,6 +19,7 @@ use execution_layer::{ use fork_choice::{ CountUnrealized, Error as ForkChoiceError, InvalidationOperation, PayloadVerificationStatus, }; +use logging::test_logger; use proto_array::{Error as ProtoArrayError, ExecutionStatus}; use slot_clock::SlotClock; use std::collections::HashMap; @@ -59,6 +60,7 @@ impl InvalidPayloadRig { let harness = BeaconChainHarness::builder(MainnetEthSpec) .spec(spec) + .logger(test_logger()) .deterministic_keypairs(VALIDATOR_COUNT) .mock_execution_layer() .fresh_ephemeral_store() @@ -976,6 +978,10 @@ async fn payload_preparation() { ) .await; + rig.harness.advance_to_slot_lookahead( + next_slot, + rig.harness.chain.config.prepare_payload_lookahead, + ); rig.harness .chain .prepare_beacon_proposer(rig.harness.chain.slot().unwrap()) @@ -1119,7 +1125,11 @@ async fn payload_preparation_before_transition_block() { .get_forkchoice_update_parameters(); rig.harness .chain - .update_execution_engine_forkchoice(current_slot, forkchoice_update_params) + .update_execution_engine_forkchoice( + current_slot, + forkchoice_update_params, + OverrideForkchoiceUpdate::Yes, + ) .await .unwrap(); diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 7cc2fd8883f..7fa73e53fd5 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -105,7 +105,7 @@ pub struct PoWBlock { pub timestamp: u64, } -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct ExecutionBlockGenerator { /* * Common database @@ -317,20 +317,12 @@ impl ExecutionBlockGenerator { && !self.blocks.contains_key(&block.parent_hash()) { return Err(format!("parent block {:?} is unknown", block.parent_hash())); - } else if let Some(hashes) = self.block_hashes.get_mut(&block.block_number()) { - hashes.push(block.block_hash()); - } else { - self.block_hashes - .insert(block.block_number(), vec![block.block_hash()]); } - self.insert_block_without_checks(block) + Ok(self.insert_block_without_checks(block)) } - pub fn insert_block_without_checks( - &mut self, - block: Block, - ) -> Result { + pub fn insert_block_without_checks(&mut self, block: Block) -> ExecutionBlockHash { let block_hash = block.block_hash(); self.block_hashes .entry(block.block_number()) @@ -338,29 +330,39 @@ impl ExecutionBlockGenerator { .push(block_hash); self.blocks.insert(block_hash, block); - Ok(block_hash) + block_hash } pub fn modify_last_block(&mut self, block_modifier: impl FnOnce(&mut Block)) { - /* FIXME(sproul): fix this - if let Some((last_block_hash, block_number)) = - self.block_hashes.keys().max().and_then(|block_number| { - self.block_hashes - .get(block_number) - .map(|block| (block, *block_number)) + if let Some(last_block_hash) = self + .block_hashes + .iter_mut() + .max_by_key(|(block_number, _)| *block_number) + .and_then(|(_, block_hashes)| { + // Remove block hash, we will re-insert with the new block hash after modifying it. + block_hashes.pop() }) { - let mut block = self.blocks.remove(last_block_hash).unwrap(); + let mut block = self.blocks.remove(&last_block_hash).unwrap(); block_modifier(&mut block); + // Update the block hash after modifying the block match &mut block { Block::PoW(b) => b.block_hash = ExecutionBlockHash::from_root(b.tree_hash_root()), Block::PoS(b) => b.block_hash = ExecutionBlockHash::from_root(b.tree_hash_root()), } - self.block_hashes.insert(block_number, block.block_hash()); - self.blocks.insert(block.block_hash(), block); + + // Update head. + if self + .head_block + .as_ref() + .map_or(true, |head| head.block_hash() == last_block_hash) + { + self.head_block = Some(block.clone()); + } + + self.insert_block_without_checks(block); } - */ } pub fn get_payload(&mut self, id: &PayloadId) -> Option> { diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 41476e0f598..d27aeb7e9f7 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -359,8 +359,7 @@ impl MockServer { .write() // The EF tests supply blocks out of order, so we must import them "without checks" and // trust they form valid chains. - .insert_block_without_checks(block) - .unwrap(); + .insert_block_without_checks(block); } pub fn get_block(&self, block_hash: ExecutionBlockHash) -> Option> { diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 14d3fbf4d13..de9ced0241b 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -1,6 +1,5 @@ //! Generic tests that make use of the (newer) `InteractiveApiTester` use crate::common::*; -use beacon_chain::proposer_prep_service::PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR; use beacon_chain::{ chain_config::ReOrgThreshold, test_utils::{AttestationStrategy, BlockStrategy}, @@ -252,7 +251,7 @@ pub async fn proposer_boost_re_org_test( } */ // Simulate the scheduled call to prepare proposers at 8 seconds into the slot. - let payload_lookahead = slot_clock.slot_duration() / PAYLOAD_PREPARATION_LOOKAHEAD_FACTOR; + let payload_lookahead = harness.chain.config.prepare_payload_lookahead; let scheduled_prepare_time = slot_clock.start_of(slot_c).unwrap() - payload_lookahead; slot_clock.set_current_time(scheduled_prepare_time); harness.chain.prepare_beacon_proposer(slot_b).await.unwrap(); diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index f107990b15c..b6e28900d37 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -750,6 +750,16 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .requires("enable-proposer-re-orgs") .takes_value(true) ) + .arg( + Arg::with_name("prepare-payload-lookahead") + .long("prepare-payload-lookahead") + .value_name("MILLISECONDS") + .help("The time before the start of a proposal slot at which payload attributes \ + should be sent. Low values are useful for execution nodes which don't \ + improve their payload after the first call, and high values are useful \ + for ensuring the EL is given ample notice. Default: 1/3 of a slot.") + .takes_value(true) + ) .arg( Arg::with_name("fork-choice-before-proposal-timeout") .long("fork-choice-before-proposal-timeout") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 97c48e03aaa..adcc56d1b0a 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -658,6 +658,11 @@ pub fn get_config( } } + client_config.chain.prepare_payload_lookahead = + clap_utils::parse_optional(cli_args, "prepare-payload-lookahead")?.unwrap_or_else(|| { + Duration::from_secs(spec.seconds_per_slot) / DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR + }); + if let Some(timeout) = clap_utils::parse_optional(cli_args, "fork-choice-before-proposal-timeout")? { diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 60823552917..ac040738099 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -133,6 +133,31 @@ fn fork_choice_before_proposal_timeout_zero() { .with_config(|config| assert_eq!(config.chain.fork_choice_before_proposal_timeout_ms, 0)); } +#[test] +fn prepare_payload_lookahead_default() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.chain.prepare_payload_lookahead, + Duration::from_secs(4), + ) + }); +} + +#[test] +fn prepare_payload_lookahead_shorter() { + CommandLineTest::new() + .flag("prepare-payload-lookahead", Some("1500")) + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.chain.prepare_payload_lookahead, + Duration::from_millis(1500) + ) + }); +} + #[test] fn paranoid_block_proposal_default() { CommandLineTest::new() From 589bebf5c202f7752eb3cf5fff53bc66310af887 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 12 Oct 2022 18:48:26 +1100 Subject: [PATCH 16/37] Fix clippy --- .../beacon_chain/src/beacon_fork_choice_store.rs | 2 +- beacon_node/client/src/builder.rs | 6 +++++- beacon_node/execution_layer/src/lib.rs | 5 +++-- .../src/test_utils/execution_block_generator.rs | 4 ++-- beacon_node/execution_layer/src/test_utils/hook.rs | 9 +-------- beacon_node/src/config.rs | 14 ++++++++++---- .../proto_array/src/proto_array_fork_choice.rs | 2 +- lighthouse/tests/beacon_node.rs | 1 + 8 files changed, 24 insertions(+), 19 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs index a410398e864..3f5bfb5b6ae 100644 --- a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs +++ b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs @@ -187,7 +187,7 @@ where root: anchor_root, }; let finalized_checkpoint = justified_checkpoint; - let justified_balances = JustifiedBalances::from_justified_state(&anchor_state)?; + let justified_balances = JustifiedBalances::from_justified_state(anchor_state)?; Ok(Self { store, diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 752ba3b7bcb..3ddd6924286 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -698,7 +698,11 @@ where runtime_context.executor.spawn( async move { let result = inner_chain - .update_execution_engine_forkchoice(current_slot, params) + .update_execution_engine_forkchoice( + current_slot, + params, + Default::default(), + ) .await; // No need to exit early if setting the head fails. It will be set again if/when the diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 29e00e47af9..f76e4b79dec 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1447,12 +1447,13 @@ mod test { #[tokio::test] async fn test_forked_terminal_block() { - let (mock, block_hash) = MockExecutionLayer::default_params() + let runtime = TestRuntime::default(); + let (mock, block_hash) = MockExecutionLayer::default_params(runtime.task_executor.clone()) .move_to_terminal_block() .produce_forked_pow_block(); assert!(mock .el - .is_valid_terminal_pow_block_hash(block_hash) + .is_valid_terminal_pow_block_hash(block_hash, &mock.spec) .await .unwrap() .unwrap()); diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 7fa73e53fd5..68fb4007358 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -259,7 +259,7 @@ impl ExecutionBlockGenerator { )?; // Insert block into block tree - let _ = self.insert_block(Block::PoW(block.clone()))?; + self.insert_block(Block::PoW(block))?; // Set head if let Some(head_total_difficulty) = @@ -296,7 +296,7 @@ impl ExecutionBlockGenerator { parent_hash, )?; - let hash = self.insert_block(Block::PoW(block.clone()))?; + let hash = self.insert_block(Block::PoW(block))?; // Set head if let Some(head_total_difficulty) = self.head_block.as_ref().and_then(|b| b.total_difficulty()) diff --git a/beacon_node/execution_layer/src/test_utils/hook.rs b/beacon_node/execution_layer/src/test_utils/hook.rs index fd82f001d3d..a3748103e3e 100644 --- a/beacon_node/execution_layer/src/test_utils/hook.rs +++ b/beacon_node/execution_layer/src/test_utils/hook.rs @@ -7,18 +7,11 @@ type ForkChoiceUpdatedHook = dyn Fn( + Send + Sync; +#[derive(Default)] pub struct Hook { forkchoice_updated: Option>, } -impl Default for Hook { - fn default() -> Self { - Hook { - forkchoice_updated: None, - } - } -} - impl Hook { pub fn on_forkchoice_updated( &self, diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index adcc56d1b0a..c98bf4fa449 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1,4 +1,6 @@ -use beacon_chain::chain_config::{ReOrgThreshold, DEFAULT_RE_ORG_THRESHOLD}; +use beacon_chain::chain_config::{ + ReOrgThreshold, DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR, DEFAULT_RE_ORG_THRESHOLD, +}; use clap::ArgMatches; use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG; use client::{ClientConfig, ClientGenesis}; @@ -17,6 +19,7 @@ use std::fs; use std::net::{IpAddr, Ipv4Addr, ToSocketAddrs}; use std::path::{Path, PathBuf}; use std::str::FromStr; +use std::time::Duration; use types::{Checkpoint, Epoch, EthSpec, Hash256, PublicKeyBytes, GRAFFITI_BYTES_LEN}; use unused_port::{unused_tcp_port, unused_udp_port}; @@ -659,9 +662,12 @@ pub fn get_config( } client_config.chain.prepare_payload_lookahead = - clap_utils::parse_optional(cli_args, "prepare-payload-lookahead")?.unwrap_or_else(|| { - Duration::from_secs(spec.seconds_per_slot) / DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR - }); + clap_utils::parse_optional(cli_args, "prepare-payload-lookahead")? + .map(Duration::from_millis) + .unwrap_or_else(|| { + Duration::from_secs(spec.seconds_per_slot) + / DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR + }); if let Some(timeout) = clap_utils::parse_optional(cli_args, "fork-choice-before-proposal-timeout")? diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index 1f53e759e65..4a13863d78f 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -415,7 +415,7 @@ impl ProtoArrayForkChoice { && ffg_competitive && participation_ok && is_weak_head) - .then(|| parent_node.root); + .then_some(parent_node.root); Ok(ProposerHead { re_org_head, diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index ac040738099..7c6f05d0a26 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -11,6 +11,7 @@ use std::path::PathBuf; use std::process::Command; use std::str::FromStr; use std::string::ToString; +use std::time::Duration; use tempfile::TempDir; use types::{Address, Checkpoint, Epoch, ExecutionBlockHash, ForkName, Hash256, MainnetEthSpec}; use unused_port::{unused_tcp_port, unused_udp_port}; From 2b9d8e6f05d48220b36c058f2d2917874f3a8b28 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 13 Oct 2022 11:03:50 +1100 Subject: [PATCH 17/37] Don't override if we aren't the proposer --- beacon_node/beacon_chain/src/beacon_chain.rs | 46 ++++++++++++++------ beacon_node/execution_layer/src/lib.rs | 16 ++++--- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 1cc20e9770c..d1193c8b9d5 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3522,6 +3522,32 @@ impl BeaconChain { && parent_node.unrealized_finalized_checkpoint == head_node.unrealized_finalized_checkpoint; + // Only attempt a re-org if we have a proposer registered for the re-org slot. + let proposing_next_slot = block_slot_ok + .then(|| { + let shuffling_decision_root = + parent_node.next_epoch_shuffling_id.shuffling_decision_block; + let proposer_index = self + .beacon_proposer_cache + .lock() + .get_slot::(shuffling_decision_root, re_org_block_slot) + .or_else(|| { + debug!( + self.log, + "Fork choice override proposer shuffling miss"; + "slot" => re_org_block_slot, + "decision_root" => ?shuffling_decision_root, + ); + None + })? + .index; + self.execution_layer + .as_ref() + .map(|el| el.has_proposer_preparation_data_blocking(proposer_index as u64)) + }) + .flatten() + .unwrap_or(false); + // FIXME(sproul): add participation check // Check that the head block arrived late and is vulnerable to a re-org. This check is only @@ -3532,23 +3558,17 @@ impl BeaconChain { let head_block_late = self.block_observed_after_attestation_deadline(head_block_root, head_node.slot); - let might_re_org = current_slot_ok && block_slot_ok && ffg_competitive && head_block_late; + let might_re_org = current_slot_ok + && block_slot_ok + && ffg_competitive + && proposing_next_slot + && head_block_late; if !might_re_org { return Ok(canonical_forkchoice_params); } - let parent_head_hash = if let Some(hash) = parent_node.execution_status.block_hash() { - hash - } else { - warn!( - self.log, - "Missing exec block hash for parent"; - "parent_root" => ?parent_node.root, - "status" => ?parent_node.execution_status, - ); - return Ok(canonical_forkchoice_params); - }; + let parent_head_hash = parent_node.execution_status.block_hash(); debug!( self.log, @@ -3560,7 +3580,7 @@ impl BeaconChain { let forkchoice_update_params = ForkchoiceUpdateParameters { head_root: parent_node.root, - head_hash: Some(parent_head_hash), + head_hash: parent_head_hash, justified_hash: canonical_forkchoice_params.justified_hash, finalized_hash: canonical_forkchoice_params.finalized_hash, }; diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index f76e4b79dec..4e083973691 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -499,6 +499,16 @@ impl ExecutionLayer { .contains_key(&proposer_index) } + /// Check if a proposer is registered as a local validator, *from a synchronous context*. + /// + /// This method MUST NOT be called from an async task. + pub fn has_proposer_preparation_data_blocking(&self, proposer_index: u64) -> bool { + self.inner + .proposer_preparation_data + .blocking_lock() + .contains_key(&proposer_index) + } + /// Returns the fee-recipient address that should be used to build a block pub async fn get_suggested_fee_recipient(&self, proposer_index: u64) -> Address { if let Some(preparation_data_entry) = @@ -1034,12 +1044,6 @@ impl ExecutionLayer { // Compute the "lookahead", the time between when the payload will be produced and now. if let Some(payload_attributes) = payload_attributes { - debug!( - self.log(), - "Sending payload attributes"; - "timestamp" => payload_attributes.timestamp, - "prev_randao" => ?payload_attributes.prev_randao, - ); if let Ok(now) = SystemTime::now().duration_since(UNIX_EPOCH) { let timestamp = Duration::from_secs(payload_attributes.timestamp); if let Some(lookahead) = timestamp.checked_sub(now) { From d42176582b75cdbee08397994c4e4c3f4734266d Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 13 Oct 2022 14:37:56 +1100 Subject: [PATCH 18/37] Resolve most remainings FIXMEs --- beacon_node/beacon_chain/src/beacon_chain.rs | 46 +++++++++++++------ beacon_node/execution_layer/src/lib.rs | 2 - .../http_api/tests/interactive_tests.rs | 3 +- consensus/fork_choice/src/fork_choice.rs | 21 ++++++++- consensus/proto_array/src/lib.rs | 4 +- .../src/proto_array_fork_choice.rs | 40 ---------------- 6 files changed, 56 insertions(+), 60 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d1193c8b9d5..fa9e84bbaf2 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3481,10 +3481,10 @@ impl BeaconChain { let head_block_root = canonical_forkchoice_params.head_root; // Load details of the head block and its parent from fork choice. - let (head_node, parent_node) = { - let mut nodes = self - .canonical_head - .fork_choice_read_lock() + let (head_node, parent_node, participation_threshold) = { + let fork_choice = self.canonical_head.fork_choice_read_lock(); + + let mut nodes = fork_choice .proto_array() .core_proto_array() .iter_nodes(&head_block_root) @@ -3498,7 +3498,11 @@ impl BeaconChain { let parent = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; let head = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; - (head, parent) + + let participation_threshold = fork_choice + .compute_participation_threshold_weight(self.config.re_org_participation_threshold); + + (head, parent, participation_threshold) }; // The slot of our potential re-org block is always 1 greater than the head block because we @@ -3506,11 +3510,20 @@ impl BeaconChain { let re_org_block_slot = head_node.slot + 1; // Suppress only during the head block's slot, or at most until the end of the next slot. - // If our proposal fails entirely we will attest to the wrong head during - // `re_org_block_slot` and only re-align with the canonical chain 500ms before the start of - // the next slot (i.e. `head_slot + 2`). - // FIXME(sproul): add some timing conditions here - let current_slot_ok = head_node.slot == current_slot || re_org_block_slot == current_slot; + // If a re-orging proposal isn't made by the `max_re_org_slot_delay` then we give up + // and allow the fork choice update for the canonical head through so that we may attest + // correctly. + let current_slot_ok = if head_node.slot == current_slot { + true + } else if re_org_block_slot == current_slot { + self.slot_clock + .seconds_from_current_slot_start(self.spec.seconds_per_slot) + .map_or(false, |delay| { + delay <= max_re_org_slot_delay(self.spec.seconds_per_slot) + }) + } else { + false + }; // Only attempt single slot re-orgs, and not at epoch boundaries. let block_slot_ok = parent_node.slot + 1 == head_node.slot @@ -3523,7 +3536,7 @@ impl BeaconChain { == head_node.unrealized_finalized_checkpoint; // Only attempt a re-org if we have a proposer registered for the re-org slot. - let proposing_next_slot = block_slot_ok + let proposing_at_re_org_slot = block_slot_ok .then(|| { let shuffling_decision_root = parent_node.next_epoch_shuffling_id.shuffling_decision_block; @@ -3548,7 +3561,13 @@ impl BeaconChain { .flatten() .unwrap_or(false); - // FIXME(sproul): add participation check + // Check that the parent's weight is greater than the participation threshold. + // If we are still in the slot of the canonical head block then only check against + // a 1x threshold as all attestations may not have arrived yet. + let participation_multiplier = if current_slot == head_node.slot { 1 } else { 2 }; + let participation_ok = participation_threshold.map_or(false, |threshold| { + parent_node.weight >= threshold.saturating_mul(participation_multiplier) + }); // Check that the head block arrived late and is vulnerable to a re-org. This check is only // a heuristic compared to the proper weight check in `get_state_for_re_org`, the reason @@ -3561,7 +3580,8 @@ impl BeaconChain { let might_re_org = current_slot_ok && block_slot_ok && ffg_competitive - && proposing_next_slot + && proposing_at_re_org_slot + && participation_ok && head_block_late; if !might_re_org { diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 4e083973691..664004dfa02 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -1052,14 +1052,12 @@ impl ExecutionLayer { lookahead, ); } else { - /* FIXME(sproul): re-enable this, or fix it debug!( self.log(), "Late payload attributes"; "timestamp" => ?timestamp, "now" => ?now, ) - */ } } } diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index de9ced0241b..e8abb6498a7 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -112,7 +112,7 @@ pub async fn proposer_boost_re_org_bad_ffg() { pub async fn proposer_boost_re_org_test( head_slot: Slot, - num_head_votes: Option, + _num_head_votes: Option, re_org_threshold: Option, should_re_org: bool, ) { @@ -299,7 +299,6 @@ pub async fn proposer_boost_re_org_test( let forkchoice_updates = forkchoice_updates.lock(); let block_a_exec_hash = block_a.message().execution_payload().unwrap().block_hash(); let block_b_exec_hash = block_b.message().execution_payload().unwrap().block_hash(); - let block_c_exec_hash = block_c.message().execution_payload().unwrap().block_hash(); let block_c_timestamp = block_c.message().execution_payload().unwrap().timestamp(); diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index ff305e9023a..cd7d97be811 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -1,7 +1,7 @@ use crate::{ForkChoiceStore, InvalidationOperation}; use proto_array::{ - Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, ParticipationThreshold, - ProposerHead, ProtoArrayForkChoice, ReOrgThreshold, + calculate_proposer_boost, Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, + ParticipationThreshold, ProposerHead, ProtoArrayForkChoice, ReOrgThreshold, }; use slog::{crit, debug, warn, Logger}; use ssz_derive::{Decode, Encode}; @@ -599,6 +599,23 @@ where .map_err(Into::into) } + /// Compute the weight corresponding to `participation_threshold`. + /// + /// This is a fraction of a single committee weight, measured approximately against + /// the justified balances, just like proposer boost. + pub fn compute_participation_threshold_weight( + &self, + participation_threshold: ParticipationThreshold, + ) -> Option + where + E: EthSpec, + { + calculate_proposer_boost::( + self.fc_store.justified_balances(), + participation_threshold.0, + ) + } + /// Return information about: /// /// - The LMD head of the chain. diff --git a/consensus/proto_array/src/lib.rs b/consensus/proto_array/src/lib.rs index a117fcded53..047c16ffc3f 100644 --- a/consensus/proto_array/src/lib.rs +++ b/consensus/proto_array/src/lib.rs @@ -6,7 +6,9 @@ mod proto_array_fork_choice; mod ssz_container; pub use crate::justified_balances::JustifiedBalances; -pub use crate::proto_array::{CountUnrealizedFull, InvalidationOperation}; +pub use crate::proto_array::{ + calculate_proposer_boost, CountUnrealizedFull, InvalidationOperation, +}; pub use crate::proto_array_fork_choice::{ Block, ExecutionStatus, ParticipationThreshold, ProposerHead, ProtoArrayForkChoice, ReOrgThreshold, diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index 4a13863d78f..d627109030f 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -168,10 +168,6 @@ where &mut self.0[i] } - pub fn iter(&self) -> impl Iterator { - self.0.iter() - } - pub fn iter_mut(&mut self) -> impl Iterator { self.0.iter_mut() } @@ -428,42 +424,6 @@ impl ProtoArrayForkChoice { }) } - /// Compute the sum of attester balances of attestations to a specific block root. - /// - /// This weight is the weight unique to the block, *not* including the weight of its ancestors. - /// - /// Any `proposer_boost` in effect is ignored: only attestations are counted. - // FIXME(sproul): consider deleting - pub fn get_block_unique_weight( - &self, - block_root: Hash256, - justified_balances: &[u64], - equivocating_indices: &BTreeSet, - ) -> Result { - let mut unique_weight = 0u64; - for (validator_index, vote) in self.votes.iter().enumerate() { - // Skip equivocating validators. - if equivocating_indices.contains(&(validator_index as u64)) { - continue; - } - - // Check the `next_root` as we care about the most recent attestations, including ones - // from the previous slot that have just been dequeued but haven't run fully through - // fork choice yet. - if vote.next_root == block_root { - let validator_balance = justified_balances - .get(validator_index) - .copied() - .unwrap_or(0); - - unique_weight = unique_weight - .checked_add(validator_balance) - .ok_or(Error::UniqueWeightOverflow(block_root))?; - } - } - Ok(unique_weight) - } - /// Returns `true` if there are any blocks in `self` with an `INVALID` execution payload status. /// /// This will operate on *all* blocks, even those that do not descend from the finalized From 1e2220954c7979431b453dbd0d9e12c1a3e5a9cc Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 14 Oct 2022 14:39:22 +1100 Subject: [PATCH 19/37] Fix async lint --- .github/custom/clippy.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/custom/clippy.toml b/.github/custom/clippy.toml index f50e35bcdfd..3ccbeee44a7 100644 --- a/.github/custom/clippy.toml +++ b/.github/custom/clippy.toml @@ -16,6 +16,7 @@ async-wrapper-methods = [ "task_executor::TaskExecutor::spawn_blocking_handle", "warp_utils::task::blocking_task", "warp_utils::task::blocking_json_task", + "beacon_chain::beacon_chain::BeaconChain::spawn_blocking_handle", "validator_client::http_api::blocking_signed_json_task", "execution_layer::test_utils::MockServer::new", "execution_layer::test_utils::MockServer::new_with_config", From fc8f6b2b24381932ccebbe42e91bc70598c945c9 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 17 Oct 2022 15:59:10 +1100 Subject: [PATCH 20/37] Moar tests --- beacon_node/beacon_chain/src/builder.rs | 11 +- beacon_node/beacon_chain/src/chain_config.rs | 6 +- beacon_node/beacon_chain/src/test_utils.rs | 89 ++++++- .../http_api/tests/interactive_tests.rs | 222 +++++++++++++++--- 4 files changed, 275 insertions(+), 53 deletions(-) diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 146e1f6bde0..64f68ab1e59 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -21,7 +21,7 @@ use fork_choice::{ForkChoice, ResetPayloadStatuses}; use futures::channel::mpsc::Sender; use operation_pool::{OperationPool, PersistedOperationPool}; use parking_lot::RwLock; -use proto_array::ReOrgThreshold; +use proto_array::{ParticipationThreshold, ReOrgThreshold}; use slasher::Slasher; use slog::{crit, error, info, Logger}; use slot_clock::{SlotClock, TestingSlotClock}; @@ -165,6 +165,15 @@ where self } + /// Sets the proposer re-org participation threshold. + pub fn proposer_re_org_participation_threshold( + mut self, + threshold: ParticipationThreshold, + ) -> Self { + self.chain_config.re_org_participation_threshold = threshold; + self + } + /// Sets the store (database). /// /// Should generally be called early in the build chain. diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index d6e6cf9f5cf..1fdd180332f 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -3,9 +3,9 @@ use serde_derive::{Deserialize, Serialize}; use std::time::Duration; use types::Checkpoint; -pub const DEFAULT_RE_ORG_THRESHOLD: ReOrgThreshold = ReOrgThreshold(10); +pub const DEFAULT_RE_ORG_THRESHOLD: ReOrgThreshold = ReOrgThreshold(20); pub const DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD: ParticipationThreshold = - ParticipationThreshold(80); + ParticipationThreshold(70); pub const DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT: u64 = 250; /// At 12s slot times, the means that the payload preparation routine will run 4s before the start @@ -57,7 +57,7 @@ pub struct ChainConfig { pub paranoid_block_proposal: bool, /// Whether to strictly count unrealized justified votes. pub count_unrealized_full: CountUnrealizedFull, - /// The offset from the start of a proposal slot at which payload attributes should be sent. + /// The offset before the start of a proposal slot at which payload attributes should be sent. /// /// Low values are useful for execution engines which don't improve their payload after the /// first call, and high values are useful for ensuring the EL is given ample notice. diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index c6c4bd04c02..63d902bcb8a 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -873,12 +873,34 @@ where head_block_root: SignedBeaconBlockHash, attestation_slot: Slot, ) -> Vec, SubnetId)>> { + self.make_unaggregated_attestations_with_limit( + attesting_validators, + state, + state_root, + head_block_root, + attestation_slot, + None, + ) + .0 + } + + pub fn make_unaggregated_attestations_with_limit( + &self, + attesting_validators: &[usize], + state: &BeaconState, + state_root: Hash256, + head_block_root: SignedBeaconBlockHash, + attestation_slot: Slot, + limit: Option, + ) -> (Vec, SubnetId)>>, Vec) { let committee_count = state.get_committee_count_at_slot(state.slot()).unwrap(); let fork = self .spec .fork_at_epoch(attestation_slot.epoch(E::slots_per_epoch())); - state + let attesters = Mutex::new(vec![]); + + let attestations = state .get_beacon_committees_at_slot(attestation_slot) .expect("should get committees") .iter() @@ -890,6 +912,15 @@ where if !attesting_validators.contains(validator_index) { return None; } + + let mut attesters = attesters.lock(); + if let Some(limit) = limit { + if attesters.len() >= limit { + return None; + } + } + attesters.push(*validator_index); + let mut attestation = self .produce_unaggregated_attestation_for_block( attestation_slot, @@ -930,9 +961,19 @@ where Some((attestation, subnet_id)) }) - .collect() + .collect::>() }) - .collect() + .collect::>(); + + let attesters = attesters.into_inner(); + if let Some(limit) = limit { + assert_eq!( + limit, + attesters.len(), + "failed to generate `limit` attestations" + ); + } + (attestations, attesters) } /// A list of sync messages for the given state. @@ -1025,13 +1066,38 @@ where block_hash: SignedBeaconBlockHash, slot: Slot, ) -> HarnessAttestations { - let unaggregated_attestations = self.make_unaggregated_attestations( + self.make_attestations_with_limit( attesting_validators, state, state_root, block_hash, slot, - ); + None, + ) + .0 + } + + /// Produce exactly `limit` attestations. + /// + /// Return attestations and vec of validator indices that attested. + pub fn make_attestations_with_limit( + &self, + attesting_validators: &[usize], + state: &BeaconState, + state_root: Hash256, + block_hash: SignedBeaconBlockHash, + slot: Slot, + limit: Option, + ) -> (HarnessAttestations, Vec) { + let (unaggregated_attestations, attesters) = self + .make_unaggregated_attestations_with_limit( + attesting_validators, + state, + state_root, + block_hash, + slot, + limit, + ); let fork = self.spec.fork_at_epoch(slot.epoch(E::slots_per_epoch())); let aggregated_attestations: Vec>> = @@ -1050,7 +1116,7 @@ where .committee .iter() .find(|&validator_index| { - if !attesting_validators.contains(validator_index) { + if !attesters.contains(validator_index) { return false; } @@ -1101,10 +1167,13 @@ where }) .collect(); - unaggregated_attestations - .into_iter() - .zip(aggregated_attestations) - .collect() + ( + unaggregated_attestations + .into_iter() + .zip(aggregated_attestations) + .collect(), + attesters, + ) } pub fn make_sync_contributions( diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index e8abb6498a7..c464c2e7217 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -1,7 +1,7 @@ //! Generic tests that make use of the (newer) `InteractiveApiTester` use crate::common::*; use beacon_chain::{ - chain_config::ReOrgThreshold, + chain_config::{ParticipationThreshold, ReOrgThreshold}, test_utils::{AttestationStrategy, BlockStrategy}, }; use eth2::types::DepositContractData; @@ -93,28 +93,126 @@ impl ForkChoiceUpdates { } } +pub struct ReOrgTest { + head_slot: Slot, + re_org_threshold: u64, + participation_threshold: u64, + percent_parent_votes: usize, + percent_empty_votes: usize, + percent_head_votes: usize, + should_re_org: bool, + misprediction: bool, +} + +impl Default for ReOrgTest { + /// Default config represents a regular easy re-org. + fn default() -> Self { + Self { + head_slot: Slot::new(30), + re_org_threshold: 20, + participation_threshold: 70, + percent_parent_votes: 100, + percent_empty_votes: 100, + percent_head_votes: 0, + should_re_org: true, + misprediction: false, + } + } +} + // Test that the beacon node will try to perform proposer boost re-orgs on late blocks when // configured. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub async fn proposer_boost_re_org_zero_weight() { - proposer_boost_re_org_test(Slot::new(30), None, Some(10), true).await + proposer_boost_re_org_test(ReOrgTest::default()).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub async fn proposer_boost_re_org_epoch_boundary() { - proposer_boost_re_org_test(Slot::new(31), None, Some(10), false).await + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(31), + should_re_org: false, + ..Default::default() + }) + .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub async fn proposer_boost_re_org_bad_ffg() { - proposer_boost_re_org_test(Slot::new(64 + 22), None, Some(10), false).await + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(64 + 22), + should_re_org: false, + ..Default::default() + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_low_total_participation() { + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(30), + percent_parent_votes: 70, + percent_empty_votes: 60, + percent_head_votes: 10, + participation_threshold: 71, + should_re_org: false, + ..Default::default() + }) + .await; } +/// Participation initially appears high at 80%, but drops off to 80+50/2 = 65% after the head +/// block. This results in a misprediction. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_participation_misprediction() { + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(30), + percent_parent_votes: 80, + percent_empty_votes: 50, + percent_head_votes: 0, + participation_threshold: 70, + should_re_org: false, + misprediction: true, + ..Default::default() + }) + .await; +} + +/// The head block is late but still receives 30% of the committee vote, leading to a misprediction. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_weight_misprediction() { + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(30), + percent_empty_votes: 70, + percent_head_votes: 30, + should_re_org: false, + misprediction: true, + ..Default::default() + }) + .await; +} + +// TODO(sproul): test current slot >1 block from head +// TODO(sproul): test parent >1 block from head + +/// Run a proposer boost re-org test. +/// +/// - `head_slot`: the slot of the canonical head to be reorged +/// - `reorg_threshold`: committee percentage value for reorging +/// - `num_empty_votes`: percentage of comm of attestations for the parent block +/// - `num_head_votes`: number of attestations for the head block +/// - `should_re_org`: whether the proposer should build on the parent rather than the head pub async fn proposer_boost_re_org_test( - head_slot: Slot, - _num_head_votes: Option, - re_org_threshold: Option, - should_re_org: bool, + ReOrgTest { + head_slot, + re_org_threshold, + participation_threshold, + percent_parent_votes, + percent_empty_votes, + percent_head_votes, + should_re_org, + misprediction, + }: ReOrgTest, ) { assert!(head_slot > 0); @@ -122,17 +220,31 @@ pub async fn proposer_boost_re_org_test( let mut spec = ForkName::Merge.make_genesis_spec(E::default_spec()); spec.terminal_total_difficulty = 1.into(); - // Validator count needs to be at least 32 or proposer boost gets set to 0 when computing - // `validator_count // 32`. - let validator_count = 32; + // Ensure there are enough validators to have `attesters_per_slot`. + let attesters_per_slot = 10; + let validator_count = E::slots_per_epoch() as usize * attesters_per_slot; let all_validators = (0..validator_count).collect::>(); - let num_initial = head_slot.as_u64() - 1; + let num_initial = head_slot.as_u64() - 2; + + // Check that the required vote percentages can be satisfied exactly using `attesters_per_slot`. + assert_eq!(100 % attesters_per_slot, 0); + let percent_per_attester = 100 / attesters_per_slot; + assert_eq!(percent_parent_votes % percent_per_attester, 0); + assert_eq!(percent_empty_votes % percent_per_attester, 0); + assert_eq!(percent_head_votes % percent_per_attester, 0); + let num_parent_votes = Some(attesters_per_slot * percent_parent_votes / 100); + let num_empty_votes = Some(attesters_per_slot * percent_empty_votes / 100); + let num_head_votes = Some(attesters_per_slot * percent_head_votes / 100); let tester = InteractiveTester::::new_with_mutator( Some(spec), validator_count, Some(Box::new(move |builder| { - builder.proposer_re_org_threshold(re_org_threshold.map(ReOrgThreshold)) + builder + .proposer_re_org_threshold(Some(ReOrgThreshold(re_org_threshold))) + .proposer_re_org_participation_threshold(ParticipationThreshold( + participation_threshold, + )) })), ) .await; @@ -204,25 +316,46 @@ pub async fn proposer_boost_re_org_test( // // A | B | - | // ^ | - | C | - let slot_a = Slot::new(num_initial); + + let slot_a = Slot::new(num_initial + 1); let slot_b = slot_a + 1; let slot_c = slot_a + 2; - let block_a_root = harness.head_block_root(); - let block_a = harness.get_block(block_a_root.into()).unwrap(); - let state_a = harness.get_current_state(); + harness.advance_slot(); + let (block_a_root, block_a, state_a) = harness + .add_block_at_slot(slot_a, harness.get_current_state()) + .await + .unwrap(); + + // Attest to block A during slot A. + let (block_a_parent_votes, _) = harness.make_attestations_with_limit( + &all_validators, + &state_a, + state_a.canonical_root(), + block_a_root, + slot_a, + num_parent_votes, + ); + harness.process_attestations(block_a_parent_votes); // Attest to block A during slot B. harness.advance_slot(); - let block_a_empty_votes = harness.make_attestations( + let (block_a_empty_votes, block_a_attesters) = harness.make_attestations_with_limit( &all_validators, &state_a, state_a.canonical_root(), - block_a_root.into(), + block_a_root, slot_b, + num_empty_votes, ); harness.process_attestations(block_a_empty_votes); + let remaining_attesters = all_validators + .iter() + .copied() + .filter(|index| !block_a_attesters.contains(index)) + .collect::>(); + // Produce block B and process it halfway through the slot. let (block_b, mut state_b) = harness.make_block(state_a.clone(), slot_b).await; let block_b_root = block_b.canonical_root(); @@ -239,21 +372,19 @@ pub async fn proposer_boost_re_org_test( harness.process_block_result(block_b.clone()).await.unwrap(); // Add attestations to block B. - /* FIXME(sproul): implement attestations - if let Some(num_head_votes) = num_head_votes { - harness.attest_block( - &state_b, - state_b.canonical_root(), - block_b_root, - &block_b, - &[] - ) - } - */ + let (block_b_head_votes, _) = harness.make_attestations_with_limit( + &remaining_attesters, + &state_b, + state_b.canonical_root(), + block_b_root.into(), + slot_b, + num_head_votes, + ); + harness.process_attestations(block_b_head_votes); + // Simulate the scheduled call to prepare proposers at 8 seconds into the slot. let payload_lookahead = harness.chain.config.prepare_payload_lookahead; - let scheduled_prepare_time = slot_clock.start_of(slot_c).unwrap() - payload_lookahead; - slot_clock.set_current_time(scheduled_prepare_time); + harness.advance_to_slot_lookahead(slot_c, payload_lookahead); harness.chain.prepare_beacon_proposer(slot_b).await.unwrap(); // Produce block C. @@ -281,7 +412,7 @@ pub async fn proposer_boost_re_org_test( if should_re_org { // Block C should build on A. - assert_eq!(block_c.parent_root(), block_a_root); + assert_eq!(block_c.parent_root(), block_a_root.into()); } else { // Block C should build on B. assert_eq!(block_c.parent_root(), block_b_root); @@ -325,12 +456,25 @@ pub async fn proposer_boost_re_org_test( .unwrap() .checked_sub(first_update.received_at) .unwrap(); - assert!( - lookahead >= payload_lookahead && lookahead <= 2 * payload_lookahead, - "lookahead={lookahead:?}, timestamp={}, prev_randao={:?}", - payload_attribs.timestamp, - payload_attribs.prev_randao, - ); + + if !misprediction { + assert!( + lookahead >= payload_lookahead && lookahead <= 2 * payload_lookahead, + "lookahead={lookahead:?}, timestamp={}, prev_randao={:?}", + payload_attribs.timestamp, + payload_attribs.prev_randao, + ); + } else { + // On a misprediction we issue the first fcU immediately before creating a block. + // This is sub-optimal and could likely be improved in future with some clever tricks. + assert_eq!( + lookahead, + Duration::from_secs(0), + "timestamp={}, prev_randao={:?}", + payload_attribs.timestamp, + payload_attribs.prev_randao, + ); + } } // Test that running fork choice before proposing results in selection of the correct head. From 158355d525f8ebf7f569878b71bbd60575082d28 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 17 Oct 2022 18:30:25 +1100 Subject: [PATCH 21/37] Check head weight 500ms before re-org slot start --- beacon_node/beacon_chain/src/beacon_chain.rs | 71 +++++++++++++------ beacon_node/beacon_chain/src/chain_config.rs | 6 +- .../beacon_chain/src/state_advance_timer.rs | 15 +++- .../http_api/tests/interactive_tests.rs | 26 +++---- consensus/fork_choice/src/fork_choice.rs | 12 +--- 5 files changed, 86 insertions(+), 44 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index fa9e84bbaf2..fe6fdb6cecc 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -54,7 +54,7 @@ use execution_layer::{ BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, PayloadAttributes, PayloadStatus, }; use fork_choice::{ - AttestationFromBlock, ExecutionStatus, ForkChoice, ForkchoiceUpdateParameters, + AttestationFromBlock, ExecutionStatus, ForkChoice, ForkChoiceStore, ForkchoiceUpdateParameters, InvalidationOperation, PayloadVerificationStatus, ResetPayloadStatuses, }; use futures::channel::mpsc::Sender; @@ -3471,17 +3471,24 @@ impl BeaconChain { fn overridden_forkchoice_update_params( &self, canonical_forkchoice_params: ForkchoiceUpdateParameters, - current_slot: Slot, ) -> Result { // Never override if proposer re-orgs are disabled. - if self.config.re_org_threshold.is_none() { + let re_org_threshold = if let Some(threshold) = self.config.re_org_threshold { + threshold + } else { return Ok(canonical_forkchoice_params); - } + }; let head_block_root = canonical_forkchoice_params.head_root; // Load details of the head block and its parent from fork choice. - let (head_node, parent_node, participation_threshold) = { + let ( + head_node, + parent_node, + participation_threshold_weight, + re_org_threshold_weight, + fork_choice_slot, + ) = { let fork_choice = self.canonical_head.fork_choice_read_lock(); let mut nodes = fork_choice @@ -3499,28 +3506,43 @@ impl BeaconChain { let parent = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; let head = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; - let participation_threshold = fork_choice - .compute_participation_threshold_weight(self.config.re_org_participation_threshold); + let participation_threshold_weight = fork_choice + .compute_committee_fraction(self.config.re_org_participation_threshold.0); - (head, parent, participation_threshold) + let re_org_threshold_weight = + fork_choice.compute_committee_fraction(re_org_threshold.0); + + let slot = fork_choice.fc_store().get_current_slot(); + + ( + head, + parent, + participation_threshold_weight, + re_org_threshold_weight, + slot, + ) }; // The slot of our potential re-org block is always 1 greater than the head block because we // only attempt single-slot re-orgs. let re_org_block_slot = head_node.slot + 1; - // Suppress only during the head block's slot, or at most until the end of the next slot. // If a re-orging proposal isn't made by the `max_re_org_slot_delay` then we give up // and allow the fork choice update for the canonical head through so that we may attest // correctly. - let current_slot_ok = if head_node.slot == current_slot { + let current_slot_ok = if head_node.slot == fork_choice_slot { true - } else if re_org_block_slot == current_slot { + } else if re_org_block_slot == fork_choice_slot { self.slot_clock - .seconds_from_current_slot_start(self.spec.seconds_per_slot) - .map_or(false, |delay| { - delay <= max_re_org_slot_delay(self.spec.seconds_per_slot) + .start_of(re_org_block_slot) + .and_then(|slot_start| { + let now = self.slot_clock.now_duration()?; + Some( + now.saturating_sub(slot_start) + <= max_re_org_slot_delay(self.spec.seconds_per_slot), + ) }) + .unwrap_or(false) } else { false }; @@ -3564,11 +3586,19 @@ impl BeaconChain { // Check that the parent's weight is greater than the participation threshold. // If we are still in the slot of the canonical head block then only check against // a 1x threshold as all attestations may not have arrived yet. - let participation_multiplier = if current_slot == head_node.slot { 1 } else { 2 }; - let participation_ok = participation_threshold.map_or(false, |threshold| { + let participation_multiplier = fork_choice_slot.saturating_sub(parent_node.slot).as_u64(); + let participation_ok = participation_threshold_weight.map_or(false, |threshold| { parent_node.weight >= threshold.saturating_mul(participation_multiplier) }); + // If the current slot is already equal to the proposal slot (or we are in the tail end of + // the prior slot), then check the actual weight of the head against the re-org threshold. + let head_weak = if fork_choice_slot == re_org_block_slot { + re_org_threshold_weight.map_or(false, |threshold| head_node.weight < threshold) + } else { + true + }; + // Check that the head block arrived late and is vulnerable to a re-org. This check is only // a heuristic compared to the proper weight check in `get_state_for_re_org`, the reason // being that we may have only *just* received the block and not yet processed any @@ -3582,6 +3612,7 @@ impl BeaconChain { && ffg_competitive && proposing_at_re_org_slot && participation_ok + && head_weak && head_block_late; if !might_re_org { @@ -3595,7 +3626,7 @@ impl BeaconChain { "Fork choice update overridden"; "canonical_head" => ?head_node.root, "override" => ?parent_node.root, - "slot" => current_slot, + "slot" => fork_choice_slot, ); let forkchoice_update_params = ForkchoiceUpdateParameters { @@ -4232,8 +4263,8 @@ impl BeaconChain { } let canonical_fcu_params = cached_head.forkchoice_update_parameters(); - let fcu_params = chain - .overridden_forkchoice_update_params(canonical_fcu_params, current_slot)?; + let fcu_params = + chain.overridden_forkchoice_update_params(canonical_fcu_params)?; let pre_payload_attributes = chain.get_pre_payload_attributes( prepare_slot, fcu_params.head_root, @@ -4370,7 +4401,7 @@ impl BeaconChain { let params = if override_forkchoice_update == OverrideForkchoiceUpdate::Yes { let chain = self.clone(); self.spawn_blocking_handle( - move || chain.overridden_forkchoice_update_params(input_params, current_slot), + move || chain.overridden_forkchoice_update_params(input_params), "update_execution_engine_forkchoice_override", ) .await?? diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index 1fdd180332f..66e74b0cc28 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -8,10 +8,12 @@ pub const DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD: ParticipationThreshold = ParticipationThreshold(70); pub const DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT: u64 = 250; -/// At 12s slot times, the means that the payload preparation routine will run 4s before the start -/// of each slot (`12 / 3 = 4`). +/// Default fraction of a slot lookahead for payload preparation (12/3 = 4 seconds on mainnet). pub const DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR: u32 = 3; +/// Fraction of a slot lookahead for fork choice in the state advance timer (500ms on mainnet). +pub const FORK_CHOICE_LOOKAHEAD_FACTOR: u32 = 24; + #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)] pub struct ChainConfig { /// Maximum number of slots to skip when importing a consensus message (e.g., block, diff --git a/beacon_node/beacon_chain/src/state_advance_timer.rs b/beacon_node/beacon_chain/src/state_advance_timer.rs index 72fc973e546..f73223fa540 100644 --- a/beacon_node/beacon_chain/src/state_advance_timer.rs +++ b/beacon_node/beacon_chain/src/state_advance_timer.rs @@ -16,6 +16,7 @@ use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS; use crate::{ beacon_chain::{ATTESTATION_CACHE_LOCK_TIMEOUT, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT}, + chain_config::FORK_CHOICE_LOOKAHEAD_FACTOR, snapshot_cache::StateAdvance, BeaconChain, BeaconChainError, BeaconChainTypes, }; @@ -133,7 +134,7 @@ async fn state_advance_timer( // Run fork choice 23/24s of the way through the slot (11.5s on mainnet). // We need to run after the state advance, so use the same condition as above. - let fork_choice_offset = slot_duration / 24; + let fork_choice_offset = slot_duration / FORK_CHOICE_LOOKAHEAD_FACTOR; let fork_choice_instant = if duration_to_next_slot > state_advance_offset { Instant::now() + duration_to_next_slot - fork_choice_offset } else { @@ -224,8 +225,20 @@ async fn state_advance_timer( return; } + // Re-compute the head, dequeuing attestations for the current slot early. beacon_chain.recompute_head_at_slot(next_slot).await; + // Prepare proposers so that the node can send payload attributes in the case where + // it decides to abandon a proposer boost re-org. + if let Err(e) = beacon_chain.prepare_beacon_proposer(current_slot).await { + warn!( + log, + "Unable to prepare proposer with lookahead"; + "error" => ?e, + "slot" => next_slot, + ); + } + // Use a blocking task to avoid blocking the core executor whilst waiting for locks // in `ForkChoiceSignalTx`. beacon_chain.task_executor.clone().spawn_blocking( diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index c464c2e7217..27156e899ce 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -192,8 +192,8 @@ pub async fn proposer_boost_re_org_weight_misprediction() { .await; } -// TODO(sproul): test current slot >1 block from head -// TODO(sproul): test parent >1 block from head +// FIXME(sproul): test current slot >1 block from head +// FIXME(sproul): test parent >1 block from head /// Run a proposer boost re-org test. /// @@ -387,6 +387,12 @@ pub async fn proposer_boost_re_org_test( harness.advance_to_slot_lookahead(slot_c, payload_lookahead); harness.chain.prepare_beacon_proposer(slot_b).await.unwrap(); + // Simulate the scheduled call to fork choice + prepare proposers 500ms before the next slot. + let fork_choice_lookahead = Duration::from_millis(500); + harness.advance_to_slot_lookahead(slot_c, fork_choice_lookahead); + harness.chain.recompute_head_at_slot(slot_c).await; + harness.chain.prepare_beacon_proposer(slot_b).await.unwrap(); + // Produce block C. while harness.get_current_slot() != slot_c { harness.advance_slot(); @@ -458,21 +464,17 @@ pub async fn proposer_boost_re_org_test( .unwrap(); if !misprediction { - assert!( - lookahead >= payload_lookahead && lookahead <= 2 * payload_lookahead, + assert_eq!( + lookahead, payload_lookahead, "lookahead={lookahead:?}, timestamp={}, prev_randao={:?}", - payload_attribs.timestamp, - payload_attribs.prev_randao, + payload_attribs.timestamp, payload_attribs.prev_randao, ); } else { - // On a misprediction we issue the first fcU immediately before creating a block. - // This is sub-optimal and could likely be improved in future with some clever tricks. + // On a misprediction we issue the first fcU 500ms before creating a block! assert_eq!( - lookahead, - Duration::from_secs(0), + lookahead, fork_choice_lookahead, "timestamp={}, prev_randao={:?}", - payload_attribs.timestamp, - payload_attribs.prev_randao, + payload_attribs.timestamp, payload_attribs.prev_randao, ); } } diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index cd7d97be811..aa77dc6ee1b 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -599,21 +599,15 @@ where .map_err(Into::into) } - /// Compute the weight corresponding to `participation_threshold`. + /// Compute the weight corresponding to `committee_percent`. /// /// This is a fraction of a single committee weight, measured approximately against /// the justified balances, just like proposer boost. - pub fn compute_participation_threshold_weight( - &self, - participation_threshold: ParticipationThreshold, - ) -> Option + pub fn compute_committee_fraction(&self, committee_percent: u64) -> Option where E: EthSpec, { - calculate_proposer_boost::( - self.fc_store.justified_balances(), - participation_threshold.0, - ) + calculate_proposer_boost::(self.fc_store.justified_balances(), committee_percent) } /// Return information about: From 113fbb45307965a5436f4fb3de265f7c5130bfcd Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 18 Oct 2022 13:29:36 +1100 Subject: [PATCH 22/37] Fix PoW fork test --- .../src/test_utils/execution_block_generator.rs | 10 +++++++++- .../src/test_utils/mock_execution_layer.rs | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 68fb4007358..22dcb400708 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -282,6 +282,7 @@ impl ExecutionBlockGenerator { pub fn insert_pow_block_by_hash( &mut self, parent_hash: ExecutionBlockHash, + unique_id: u64, ) -> Result { let parent_block = self.block_by_hash(parent_hash).ok_or_else(|| { format!( @@ -289,14 +290,21 @@ impl ExecutionBlockGenerator { parent_hash ) })?; - let block = generate_pow_block( + + let mut block = generate_pow_block( self.terminal_total_difficulty, self.terminal_block_number, parent_block.block_number() + 1, parent_hash, )?; + // Hack the block hash to make this block distinct from any other block with a different + // `unique_id` (the default is 0). + block.block_hash = ExecutionBlockHash::from_root(Hash256::from_low_u64_be(unique_id)); + block.block_hash = ExecutionBlockHash::from_root(block.tree_hash_root()); + let hash = self.insert_block(Block::PoW(block))?; + // Set head if let Some(head_total_difficulty) = self.head_block.as_ref().and_then(|b| b.total_difficulty()) diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index be24b3b14d4..e9d4b2121be 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -244,7 +244,7 @@ impl MockExecutionLayer { let block_hash = self .server .execution_block_generator() - .insert_pow_block_by_hash(head_block.parent_hash()) + .insert_pow_block_by_hash(head_block.parent_hash(), 1) .unwrap(); (self, block_hash) } From df2b3f7434420d535e46ba20ead8f5644267731f Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 18 Oct 2022 16:09:59 +1100 Subject: [PATCH 23/37] Test slot distance conditions --- .../http_api/tests/interactive_tests.rs | 90 +++++++++++++++---- 1 file changed, 73 insertions(+), 17 deletions(-) diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 27156e899ce..baa036ce2a1 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -95,6 +95,10 @@ impl ForkChoiceUpdates { pub struct ReOrgTest { head_slot: Slot, + /// Number of slots between parent block and canonical head. + parent_distance: u64, + /// Number of slots between head block and block proposal slot. + head_distance: u64, re_org_threshold: u64, participation_threshold: u64, percent_parent_votes: usize, @@ -109,6 +113,8 @@ impl Default for ReOrgTest { fn default() -> Self { Self { head_slot: Slot::new(30), + parent_distance: 1, + head_distance: 1, re_org_threshold: 20, participation_threshold: 70, percent_parent_votes: 100, @@ -161,6 +167,43 @@ pub async fn proposer_boost_re_org_low_total_participation() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_parent_distance() { + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(30), + parent_distance: 2, + should_re_org: false, + ..Default::default() + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_head_distance() { + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(29), + head_distance: 2, + should_re_org: false, + ..Default::default() + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_very_unhealthy() { + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(31), + parent_distance: 2, + head_distance: 2, + percent_parent_votes: 10, + percent_empty_votes: 10, + percent_head_votes: 10, + should_re_org: false, + ..Default::default() + }) + .await; +} + /// Participation initially appears high at 80%, but drops off to 80+50/2 = 65% after the head /// block. This results in a misprediction. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -192,9 +235,6 @@ pub async fn proposer_boost_re_org_weight_misprediction() { .await; } -// FIXME(sproul): test current slot >1 block from head -// FIXME(sproul): test parent >1 block from head - /// Run a proposer boost re-org test. /// /// - `head_slot`: the slot of the canonical head to be reorged @@ -205,6 +245,8 @@ pub async fn proposer_boost_re_org_weight_misprediction() { pub async fn proposer_boost_re_org_test( ReOrgTest { head_slot, + parent_distance, + head_distance, re_org_threshold, participation_threshold, percent_parent_votes, @@ -224,7 +266,7 @@ pub async fn proposer_boost_re_org_test( let attesters_per_slot = 10; let validator_count = E::slots_per_epoch() as usize * attesters_per_slot; let all_validators = (0..validator_count).collect::>(); - let num_initial = head_slot.as_u64() - 2; + let num_initial = head_slot.as_u64().checked_sub(parent_distance + 1).unwrap(); // Check that the required vote percentages can be satisfied exactly using `attesters_per_slot`. assert_eq!(100 % attesters_per_slot, 0); @@ -318,8 +360,8 @@ pub async fn proposer_boost_re_org_test( // ^ | - | C | let slot_a = Slot::new(num_initial + 1); - let slot_b = slot_a + 1; - let slot_c = slot_a + 2; + let slot_b = slot_a + parent_distance; + let slot_c = slot_b + head_distance; harness.advance_slot(); let (block_a_root, block_a, state_a) = harness @@ -339,7 +381,9 @@ pub async fn proposer_boost_re_org_test( harness.process_attestations(block_a_parent_votes); // Attest to block A during slot B. - harness.advance_slot(); + for _ in 0..parent_distance { + harness.advance_slot(); + } let (block_a_empty_votes, block_a_attesters) = harness.make_attestations_with_limit( &all_validators, &state_a, @@ -382,23 +426,35 @@ pub async fn proposer_boost_re_org_test( ); harness.process_attestations(block_b_head_votes); - // Simulate the scheduled call to prepare proposers at 8 seconds into the slot. let payload_lookahead = harness.chain.config.prepare_payload_lookahead; - harness.advance_to_slot_lookahead(slot_c, payload_lookahead); - harness.chain.prepare_beacon_proposer(slot_b).await.unwrap(); - - // Simulate the scheduled call to fork choice + prepare proposers 500ms before the next slot. let fork_choice_lookahead = Duration::from_millis(500); - harness.advance_to_slot_lookahead(slot_c, fork_choice_lookahead); - harness.chain.recompute_head_at_slot(slot_c).await; - harness.chain.prepare_beacon_proposer(slot_b).await.unwrap(); - - // Produce block C. while harness.get_current_slot() != slot_c { + let current_slot = harness.get_current_slot(); + let next_slot = current_slot + 1; + + // Simulate the scheduled call to prepare proposers at 8 seconds into the slot. + harness.advance_to_slot_lookahead(next_slot, payload_lookahead); + harness + .chain + .prepare_beacon_proposer(current_slot) + .await + .unwrap(); + + // Simulate the scheduled call to fork choice + prepare proposers 500ms before the + // next slot. + harness.advance_to_slot_lookahead(next_slot, fork_choice_lookahead); + harness.chain.recompute_head_at_slot(next_slot).await; + harness + .chain + .prepare_beacon_proposer(current_slot) + .await + .unwrap(); + harness.advance_slot(); harness.chain.per_slot_task().await; } + // Produce block C. // Advance state_b so we can get the proposer. complete_state_advance(&mut state_b, None, slot_c, &harness.chain.spec).unwrap(); From a416f9c5ff98009c571713f65233554594deb4b4 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 18 Oct 2022 17:28:12 +1100 Subject: [PATCH 24/37] Clean up CLI --- beacon_node/src/cli.rs | 28 ++++++++++++------- beacon_node/src/config.rs | 25 +++++++++-------- lighthouse/tests/beacon_node.rs | 41 +++++++++++++++------------- scripts/local_testnet/beacon_node.sh | 4 +-- scripts/local_testnet/vars.env | 6 ---- 5 files changed, 55 insertions(+), 49 deletions(-) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 9a8a9aa2271..26b69dc3ea6 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -747,18 +747,26 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(false) ) .arg( - Arg::with_name("enable-proposer-re-orgs") - .long("enable-proposer-re-orgs") - .help("Attempt to re-org out weak/late blocks from other proposers \ - (potentially risky)") - .takes_value(true) + Arg::with_name("disable-proposer-reorgs") + .long("disable-proposer-reorgs") + .help("Do not attempt to reorg late blocks from other validators when proposing.") + .takes_value(false) ) .arg( - Arg::with_name("proposer-re-org-fraction") - .long("proposer-re-org-fraction") - .help("Percentage of vote weight below which to attempt a proposer re-org") - .requires("enable-proposer-re-orgs") - .takes_value(true) + Arg::with_name("proposer-reorg-threshold") + .long("proposer-reorg-threshold") + .value_name("PERCENT") + .help("Percentage of vote weight below which to attempt a proposer reorg. \ + Default: 20%") + .conflicts_with("disable-proposer-reorgs") + ) + .arg( + Arg::with_name("proposer-reorg-participation-threshold") + .long("proposer-reorg-participation-threshold") + .value_name("PERCENT") + .help("Minimum participation percentage at proposer reorgs are allowed. \ + Default: 70%") + .conflicts_with("disable-proposer-reorgs") ) .arg( Arg::with_name("prepare-payload-lookahead") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index aa620083f0e..ead057d1f5e 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1,5 +1,6 @@ use beacon_chain::chain_config::{ - ReOrgThreshold, DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR, DEFAULT_RE_ORG_THRESHOLD, + ParticipationThreshold, ReOrgThreshold, DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR, + DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD, DEFAULT_RE_ORG_THRESHOLD, }; use clap::ArgMatches; use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG; @@ -669,16 +670,18 @@ pub fn get_config( client_config.chain.enable_lock_timeouts = false; } - if let Some(enable_re_orgs) = clap_utils::parse_optional(cli_args, "enable-proposer-re-orgs")? { - if enable_re_orgs { - client_config.chain.re_org_threshold = Some( - clap_utils::parse_optional(cli_args, "proposer-re-org-fraction")? - .map(ReOrgThreshold) - .unwrap_or(DEFAULT_RE_ORG_THRESHOLD), - ); - } else { - client_config.chain.re_org_threshold = None; - } + if cli_args.is_present("disable-proposer-reorgs") { + client_config.chain.re_org_threshold = None; + } else { + client_config.chain.re_org_threshold = Some( + clap_utils::parse_optional(cli_args, "proposer-reorg-threshold")? + .map(ReOrgThreshold) + .unwrap_or(DEFAULT_RE_ORG_THRESHOLD), + ); + client_config.chain.re_org_participation_threshold = + clap_utils::parse_optional(cli_args, "proposer-reorg-participation-threshold")? + .map(ParticipationThreshold) + .unwrap_or(DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD); } client_config.chain.prepare_payload_lookahead = diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 8d1190d611a..c9bee37e677 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -1,7 +1,9 @@ use beacon_node::{beacon_chain::CountUnrealizedFull, ClientConfig as Config}; use crate::exec::{CommandLineTestExec, CompletedTest}; -use beacon_node::beacon_chain::chain_config::DEFAULT_RE_ORG_THRESHOLD; +use beacon_node::beacon_chain::chain_config::{ + DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD, DEFAULT_RE_ORG_THRESHOLD, +}; use eth1::Eth1Endpoint; use lighthouse_network::PeerId; use std::fs::File; @@ -1491,40 +1493,41 @@ fn ensure_panic_on_failed_launch() { } #[test] -fn enable_proposer_re_orgs_true() { - CommandLineTest::new() - .flag("enable-proposer-re-orgs", Some("true")) - .run() - .with_config(|config| { - assert_eq!( - config.chain.re_org_threshold, - Some(DEFAULT_RE_ORG_THRESHOLD) - ) - }); +fn enable_proposer_re_orgs_default() { + CommandLineTest::new().run().with_config(|config| { + assert_eq!( + config.chain.re_org_threshold, + Some(DEFAULT_RE_ORG_THRESHOLD) + ); + assert_eq!( + config.chain.re_org_participation_threshold, + DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD, + ); + }); } #[test] -fn enable_proposer_re_orgs_false() { +fn disable_proposer_re_orgs() { CommandLineTest::new() - .flag("enable-proposer-re-orgs", Some("false")) + .flag("disable-proposer-reorgs", None) .run() .with_config(|config| assert_eq!(config.chain.re_org_threshold, None)); } #[test] -fn enable_proposer_re_orgs_default() { +fn proposer_re_org_threshold() { CommandLineTest::new() + .flag("proposer-reorg-threshold", Some("90")) .run() - .with_config(|config| assert_eq!(config.chain.re_org_threshold, None)); + .with_config(|config| assert_eq!(config.chain.re_org_threshold.unwrap().0, 90)); } #[test] -fn proposer_re_org_fraction() { +fn proposer_re_org_participation_threshold() { CommandLineTest::new() - .flag("enable-proposer-re-orgs", Some("true")) - .flag("proposer-re-org-fraction", Some("90")) + .flag("proposer-reorg-participation-threshold", Some("20")) .run() - .with_config(|config| assert_eq!(config.chain.re_org_threshold.unwrap().0, 90)); + .with_config(|config| assert_eq!(config.chain.re_org_participation_threshold.0, 20)); } #[test] diff --git a/scripts/local_testnet/beacon_node.sh b/scripts/local_testnet/beacon_node.sh index 55e8874fa72..ac61b54dfb3 100755 --- a/scripts/local_testnet/beacon_node.sh +++ b/scripts/local_testnet/beacon_node.sh @@ -54,6 +54,4 @@ exec lighthouse \ --port $network_port \ --http-port $http_port \ --disable-packet-filter \ - --target-peers $((BN_COUNT - 1)) \ - --enable-proposer-re-orgs "$ENABLE_PROPOSER_RE_ORGS" \ - --proposer-re-org-fraction "$PROPOSER_RE_ORG_FRACTION" + --target-peers $((BN_COUNT - 1)) diff --git a/scripts/local_testnet/vars.env b/scripts/local_testnet/vars.env index 6ecb672022f..2506e9e1cdf 100644 --- a/scripts/local_testnet/vars.env +++ b/scripts/local_testnet/vars.env @@ -47,11 +47,5 @@ SECONDS_PER_ETH1_BLOCK=1 # Proposer score boost percentage PROPOSER_SCORE_BOOST=70 -# Enable re-orgs of late blocks using the proposer boost? -ENABLE_PROPOSER_RE_ORGS=true - -# Minimum percentage of the vote that a block must have in order to not get re-orged. -PROPOSER_RE_ORG_FRACTION=50 - # Command line arguments for validator client VC_ARGS="" From 725de699c08474039e2efb7acd8ddc64fdcd401b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 18 Oct 2022 17:54:25 +1100 Subject: [PATCH 25/37] Self-review --- beacon_node/beacon_chain/src/beacon_chain.rs | 3 +++ beacon_node/beacon_chain/src/chain_config.rs | 2 +- consensus/proto_array/src/error.rs | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e5bbfcf3627..894fd0a33f1 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3372,7 +3372,10 @@ impl BeaconChain { None } + /// Get the proposer index and `prev_randao` value for a proposal at slot `proposal_slot`. /// + /// The `proposer_head` may be the head block of `cached_head` or its parent. An error will + /// be returned for any other value. pub fn get_pre_payload_attributes( &self, proposal_slot: Slot, diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index 66e74b0cc28..62959ce3ce0 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -74,7 +74,7 @@ impl Default for ChainConfig { reconstruct_historic_states: false, enable_lock_timeouts: true, max_network_size: 10 * 1_048_576, // 10M - re_org_threshold: None, + re_org_threshold: Some(DEFAULT_RE_ORG_THRESHOLD), re_org_participation_threshold: DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD, fork_choice_before_proposal_timeout_ms: DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT, // Builder fallback configs that are set in `clap` will override these. diff --git a/consensus/proto_array/src/error.rs b/consensus/proto_array/src/error.rs index d2fdad56a77..e58c8faf299 100644 --- a/consensus/proto_array/src/error.rs +++ b/consensus/proto_array/src/error.rs @@ -16,7 +16,6 @@ pub enum Error { InvalidNodeDelta(usize), DeltaOverflow(usize), ProposerBoostOverflow(usize), - UniqueWeightOverflow(Hash256), IndexOverflow(&'static str), InvalidExecutionDeltaOverflow(usize), InvalidDeltaLen { From f30f2fedc92b2b08c9cca33d2f701343729d5836 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 19 Oct 2022 10:30:02 +1100 Subject: [PATCH 26/37] Address Sean's review comments --- beacon_node/beacon_chain/src/beacon_chain.rs | 21 ++++++++++--------- consensus/fork_choice/src/fork_choice.rs | 6 +++--- .../execution_status.rs | 2 +- consensus/proto_array/src/lib.rs | 2 +- consensus/proto_array/src/proto_array.rs | 10 +++++---- .../src/proto_array_fork_choice.rs | 18 +++++++++------- 6 files changed, 32 insertions(+), 27 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 894fd0a33f1..a21bfca85d4 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -93,6 +93,7 @@ use store::{ use task_executor::{ShutdownReason, TaskExecutor}; use tree_hash::TreeHash; use types::beacon_state::CloneConfig; +use types::consts::merge::INTERVALS_PER_SLOT; use types::*; pub use crate::canonical_head::{CanonicalHead, CanonicalHeadRwLock}; @@ -117,7 +118,7 @@ pub const VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT: Duration = Duration::from_secs(1) /// The latest delay from the start of the slot at which to attempt a 1-slot re-org. fn max_re_org_slot_delay(seconds_per_slot: u64) -> Duration { // Allow at least half of the attestation deadline for the block to propagate. - Duration::from_secs(seconds_per_slot) / 6 + Duration::from_secs(seconds_per_slot) / INTERVALS_PER_SLOT as u32 / 2 } // These keys are all zero because they get stored in different columns, see `DBColumn` type. @@ -3226,7 +3227,7 @@ impl BeaconChain { self.log, "Proposing block to re-org current head"; "slot" => slot, - "head" => %head_block_root, + "head_to_reorg" => %head_block_root, ); (re_org_state.pre_state, re_org_state.state_root) } @@ -3311,7 +3312,7 @@ impl BeaconChain { let proposing_on_time = slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot); let head_late = self.block_observed_after_attestation_deadline(canonical_head, head_slot); let mut proposer_head = Default::default(); - let mut cache_hit = true; + let mut cache_miss = false; if proposing_on_time && head_late { // Is the current head weak and appropriate for re-orging? @@ -3346,13 +3347,13 @@ impl BeaconChain { self.log, "Attempting re-org due to weak head"; "head" => ?canonical_head, - "re_org_head" => ?re_org_head, - "head_weight" => ?proposer_head.canonical_head_weight, - "re_org_weight" => ?proposer_head.re_org_weight_threshold, + "parent" => ?re_org_head, + "head_weight" => proposer_head.canonical_head_weight.unwrap_or(0), + "threshold_weight" => proposer_head.re_org_weight_threshold.unwrap_or(0), ); return Some(pre_state); } - cache_hit = false; + cache_miss = true; } } debug!( @@ -3365,7 +3366,7 @@ impl BeaconChain { "proposing_on_time" => proposing_on_time, "single_slot" => proposer_head.is_single_slot_re_org, "ffg_competitive" => proposer_head.ffg_competitive, - "cache_hit" => cache_hit, + "cache_miss" => cache_miss, "shuffling_stable" => proposer_head.shuffling_stable, "participation_ok" => proposer_head.participation_ok, ); @@ -3525,10 +3526,10 @@ impl BeaconChain { let head = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; let participation_threshold_weight = fork_choice - .compute_committee_fraction(self.config.re_org_participation_threshold.0); + .calculate_committee_fraction(self.config.re_org_participation_threshold.0); let re_org_threshold_weight = - fork_choice.compute_committee_fraction(re_org_threshold.0); + fork_choice.calculate_committee_fraction(re_org_threshold.0); let slot = fork_choice.fc_store().get_current_slot(); diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index aa77dc6ee1b..4278d8f0b6f 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -1,6 +1,6 @@ use crate::{ForkChoiceStore, InvalidationOperation}; use proto_array::{ - calculate_proposer_boost, Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, + calculate_committee_fraction, Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, ParticipationThreshold, ProposerHead, ProtoArrayForkChoice, ReOrgThreshold, }; use slog::{crit, debug, warn, Logger}; @@ -603,11 +603,11 @@ where /// /// This is a fraction of a single committee weight, measured approximately against /// the justified balances, just like proposer boost. - pub fn compute_committee_fraction(&self, committee_percent: u64) -> Option + pub fn calculate_committee_fraction(&self, committee_percent: u64) -> Option where E: EthSpec, { - calculate_proposer_boost::(self.fc_store.justified_balances(), committee_percent) + calculate_committee_fraction::(self.fc_store.justified_balances(), committee_percent) } /// Return information about: diff --git a/consensus/proto_array/src/fork_choice_test_definition/execution_status.rs b/consensus/proto_array/src/fork_choice_test_definition/execution_status.rs index f1b0e512d7d..ede5bb39481 100644 --- a/consensus/proto_array/src/fork_choice_test_definition/execution_status.rs +++ b/consensus/proto_array/src/fork_choice_test_definition/execution_status.rs @@ -999,7 +999,7 @@ pub fn get_execution_status_test_definition_03() -> ForkChoiceTestDefinition { }); ops.push(Operation::AssertWeight { block_root: get_root(3), - // This is a "magic number" generated from `calculate_proposer_boost`. + // This is a "magic number" generated from `calculate_committee_fraction`. weight: 31_000, }); diff --git a/consensus/proto_array/src/lib.rs b/consensus/proto_array/src/lib.rs index 047c16ffc3f..e3520322698 100644 --- a/consensus/proto_array/src/lib.rs +++ b/consensus/proto_array/src/lib.rs @@ -7,7 +7,7 @@ mod ssz_container; pub use crate::justified_balances::JustifiedBalances; pub use crate::proto_array::{ - calculate_proposer_boost, CountUnrealizedFull, InvalidationOperation, + calculate_committee_fraction, CountUnrealizedFull, InvalidationOperation, }; pub use crate::proto_array_fork_choice::{ Block, ExecutionStatus, ParticipationThreshold, ProposerHead, ProtoArrayForkChoice, diff --git a/consensus/proto_array/src/proto_array.rs b/consensus/proto_array/src/proto_array.rs index 3ce4bf65b63..add84f54787 100644 --- a/consensus/proto_array/src/proto_array.rs +++ b/consensus/proto_array/src/proto_array.rs @@ -241,9 +241,11 @@ impl ProtoArray { // Invalid nodes (or their ancestors) should not receive a proposer boost. && !execution_status_is_invalid { - proposer_score = - calculate_proposer_boost::(new_justified_balances, proposer_score_boost) - .ok_or(Error::ProposerBoostOverflow(node_index))?; + proposer_score = calculate_committee_fraction::( + new_justified_balances, + proposer_score_boost, + ) + .ok_or(Error::ProposerBoostOverflow(node_index))?; node_delta = node_delta .checked_add(proposer_score as i64) .ok_or(Error::DeltaOverflow(node_index))?; @@ -1009,7 +1011,7 @@ impl ProtoArray { /// A helper method to calculate the proposer boost based on the given `justified_balances`. /// /// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/fork-choice.md#get_latest_attesting_balance -pub fn calculate_proposer_boost( +pub fn calculate_committee_fraction( justified_balances: &JustifiedBalances, proposer_score_boost: u64, ) -> Option { diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index d627109030f..145cab17f83 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -1,8 +1,8 @@ use crate::{ error::Error, proto_array::{ - calculate_proposer_boost, CountUnrealizedFull, InvalidationOperation, Iter, ProposerBoost, - ProtoArray, ProtoNode, + calculate_committee_fraction, CountUnrealizedFull, InvalidationOperation, Iter, + ProposerBoost, ProtoArray, ProtoNode, }, ssz_container::SszContainer, JustifiedBalances, @@ -376,7 +376,7 @@ impl ProtoArrayForkChoice { // Do not re-org on the first slot of an epoch because this is liable to change the // shuffling and rob us of a proposal entirely. A more sophisticated check could be - // done here, but we're prioristing speed and simplicity over precision. + // done here, but we're prioritising speed and simplicity over precision. let shuffling_stable = current_slot % E::slots_per_epoch() != 0; // Only re-org if the new head will be competitive with the current head's justification and @@ -391,7 +391,7 @@ impl ProtoArrayForkChoice { // To prevent excessive re-orgs when the chain is struggling, only re-org when participation // is above the configured threshold. This should not overflow. let participation_committee_threshold = - calculate_proposer_boost::(justified_balances, participation_threshold.0) + calculate_committee_fraction::(justified_balances, participation_threshold.0) .ok_or_else(|| { "overflow calculating committee weight for participation threshold".to_string() })?; @@ -400,7 +400,7 @@ impl ProtoArrayForkChoice { // Only re-org if the head's weight is less than the configured committee fraction. let re_org_weight_threshold = - calculate_proposer_boost::(justified_balances, re_org_threshold.0).ok_or_else( + calculate_committee_fraction::(justified_balances, re_org_threshold.0).ok_or_else( || "overflow calculating committee weight for re-org threshold".to_string(), )?; let canonical_head_weight = head_node.weight; @@ -486,9 +486,11 @@ impl ProtoArrayForkChoice { // Compute the score based upon the current balances. We can't rely on // the `previous_proposr_boost.score` since it is set to zero with an // invalid node. - let proposer_score = - calculate_proposer_boost::(&self.balances, proposer_score_boost) - .ok_or("Failed to compute proposer boost")?; + let proposer_score = calculate_committee_fraction::( + &self.balances, + proposer_score_boost, + ) + .ok_or("Failed to compute proposer boost")?; // Store the score we've applied here so it can be removed in // a later call to `apply_score_changes`. self.proto_array.previous_proposer_boost.score = proposer_score; From be51e24701a25bf73d75b2bb9d1f9447dc198cec Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 28 Oct 2022 18:12:33 +1100 Subject: [PATCH 27/37] Consolidate checks and make them short-circuit --- beacon_node/beacon_chain/src/beacon_chain.rs | 334 +++++++++--------- beacon_node/beacon_chain/src/errors.rs | 2 +- consensus/fork_choice/src/fork_choice.rs | 63 ++-- consensus/proto_array/src/error.rs | 2 + consensus/proto_array/src/lib.rs | 4 +- .../src/proto_array_fork_choice.rs | 279 +++++++++++---- 6 files changed, 427 insertions(+), 257 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a21bfca85d4..2702aae5902 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -54,7 +54,7 @@ use execution_layer::{ BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, PayloadAttributes, PayloadStatus, }; use fork_choice::{ - AttestationFromBlock, ExecutionStatus, ForkChoice, ForkChoiceStore, ForkchoiceUpdateParameters, + AttestationFromBlock, ExecutionStatus, ForkChoice, ForkchoiceUpdateParameters, InvalidationOperation, PayloadVerificationStatus, ResetPayloadStatuses, }; use futures::channel::mpsc::Sender; @@ -62,7 +62,7 @@ use itertools::process_results; use itertools::Itertools; use operation_pool::{AttestationRef, OperationPool, PersistedOperationPool}; use parking_lot::{Mutex, RwLock}; -use proto_array::CountUnrealizedFull; +use proto_array::{CountUnrealizedFull, DoNotReOrg, ProposerHeadError}; use safe_arith::SafeArith; use slasher::Slasher; use slog::{crit, debug, error, info, trace, warn, Logger}; @@ -3310,67 +3310,81 @@ impl BeaconChain { // 2. The current head block was seen late. // 3. The `get_proposer_head` conditions from fork choice pass. let proposing_on_time = slot_delay < max_re_org_slot_delay(self.spec.seconds_per_slot); + if !proposing_on_time { + debug!( + self.log, + "Not attempting re-org"; + "reason" => "not proposing on time", + ); + return None; + } + let head_late = self.block_observed_after_attestation_deadline(canonical_head, head_slot); - let mut proposer_head = Default::default(); - let mut cache_miss = false; + if !head_late { + debug!( + self.log, + "Not attempting re-org"; + "reason" => "head not late" + ); + return None; + } - if proposing_on_time && head_late { - // Is the current head weak and appropriate for re-orging? - proposer_head = self - .canonical_head - .fork_choice_read_lock() - .get_proposer_head( - slot, - canonical_head, - re_org_threshold, - self.config.re_org_participation_threshold, - ) - .map_err(|e| { - warn!( + // Is the current head weak and appropriate for re-orging? + let proposer_head = self + .canonical_head + .fork_choice_read_lock() + .get_proposer_head( + slot, + canonical_head, + re_org_threshold, + self.config.re_org_participation_threshold, + ) + .map_err(|e| match e { + ProposerHeadError::DoNotReOrg(reason) => { + debug!( self.log, "Not attempting re-org"; - "error" => ?e, + "reason" => %reason, ); - }) - .ok()?; - - if let Some(re_org_head) = proposer_head.re_org_head { - // Only attempt a re-org if we hit the snapshot cache. - if let Some(pre_state) = self - .snapshot_cache - .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .and_then(|snapshot_cache| { - snapshot_cache.get_state_for_block_production(re_org_head) - }) - { - info!( + } + ProposerHeadError::Error(e) => { + warn!( self.log, - "Attempting re-org due to weak head"; - "head" => ?canonical_head, - "parent" => ?re_org_head, - "head_weight" => proposer_head.canonical_head_weight.unwrap_or(0), - "threshold_weight" => proposer_head.re_org_weight_threshold.unwrap_or(0), + "Not attempting re-org"; + "error" => ?e, ); - return Some(pre_state); } - cache_miss = true; - } - } - debug!( + }) + .ok()?; + let re_org_parent_block = proposer_head.parent_node.root; + + // Only attempt a re-org if we hit the snapshot cache. + let pre_state = self + .snapshot_cache + .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) + .and_then(|snapshot_cache| { + snapshot_cache.get_state_for_block_production(re_org_parent_block) + }) + .or_else(|| { + debug!( + self.log, + "Not attempting re-org"; + "reason" => "missed snapshot cache", + "parent_block" => ?re_org_parent_block, + ); + None + })?; + + info!( self.log, - "Not attempting re-org"; - "head" => ?canonical_head, - "head_weight" => ?proposer_head.canonical_head_weight, - "re_org_weight" => ?proposer_head.re_org_weight_threshold, - "head_late" => head_late, - "proposing_on_time" => proposing_on_time, - "single_slot" => proposer_head.is_single_slot_re_org, - "ffg_competitive" => proposer_head.ffg_competitive, - "cache_miss" => cache_miss, - "shuffling_stable" => proposer_head.shuffling_stable, - "participation_ok" => proposer_head.participation_ok, + "Attempting re-org due to weak head"; + "weak_head" => ?canonical_head, + "parent" => ?re_org_parent_block, + "head_weight" => proposer_head.head_node.weight, + "threshold_weight" => proposer_head.re_org_weight_threshold ); - None + + Some(pre_state) } /// Get the proposer index and `prev_randao` value for a proposal at slot `proposal_slot`. @@ -3491,132 +3505,132 @@ impl BeaconChain { &self, canonical_forkchoice_params: ForkchoiceUpdateParameters, ) -> Result { + self.overridden_forkchoice_update_params_or_failure_reason(&canonical_forkchoice_params) + .or_else(|e| match e { + ProposerHeadError::DoNotReOrg(reason) => { + trace!( + self.log, + "Not suppressing fork choice update"; + "reason" => %reason, + ); + Ok(canonical_forkchoice_params) + } + ProposerHeadError::Error(e) => Err(e), + }) + } + + fn overridden_forkchoice_update_params_or_failure_reason( + &self, + canonical_forkchoice_params: &ForkchoiceUpdateParameters, + ) -> Result> { // Never override if proposer re-orgs are disabled. - let re_org_threshold = if let Some(threshold) = self.config.re_org_threshold { - threshold - } else { - return Ok(canonical_forkchoice_params); - }; + let re_org_threshold = self + .config + .re_org_threshold + .ok_or(DoNotReOrg::ReOrgsDisabled)?; let head_block_root = canonical_forkchoice_params.head_root; - // Load details of the head block and its parent from fork choice. - let ( - head_node, - parent_node, - participation_threshold_weight, - re_org_threshold_weight, - fork_choice_slot, - ) = { - let fork_choice = self.canonical_head.fork_choice_read_lock(); - - let mut nodes = fork_choice - .proto_array() - .core_proto_array() - .iter_nodes(&head_block_root) - .take(2) - .cloned() - .collect::>(); - - if nodes.len() != 2 { - return Ok(canonical_forkchoice_params); - } - - let parent = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; - let head = nodes.pop().ok_or(Error::SuppressForkChoiceError)?; - - let participation_threshold_weight = fork_choice - .calculate_committee_fraction(self.config.re_org_participation_threshold.0); - - let re_org_threshold_weight = - fork_choice.calculate_committee_fraction(re_org_threshold.0); - - let slot = fork_choice.fc_store().get_current_slot(); - - ( - head, - parent, - participation_threshold_weight, - re_org_threshold_weight, - slot, + // Perform initial checks and load the relevant info from fork choice. + let info = self + .canonical_head + .fork_choice_read_lock() + .get_preliminary_proposer_head( + head_block_root, + re_org_threshold, + self.config.re_org_participation_threshold, ) - }; + .map_err(|e| e.map_inner_error(Error::ProposerHeadForkChoiceError))?; // The slot of our potential re-org block is always 1 greater than the head block because we // only attempt single-slot re-orgs. - let re_org_block_slot = head_node.slot + 1; + let head_slot = info.head_node.slot; + let re_org_block_slot = head_slot + 1; + let fork_choice_slot = info.current_slot; // If a re-orging proposal isn't made by the `max_re_org_slot_delay` then we give up // and allow the fork choice update for the canonical head through so that we may attest // correctly. - let current_slot_ok = if head_node.slot == fork_choice_slot { + let current_slot_ok = if head_slot == fork_choice_slot { true } else if re_org_block_slot == fork_choice_slot { self.slot_clock .start_of(re_org_block_slot) .and_then(|slot_start| { let now = self.slot_clock.now_duration()?; - Some( - now.saturating_sub(slot_start) - <= max_re_org_slot_delay(self.spec.seconds_per_slot), - ) + let slot_delay = now.saturating_sub(slot_start); + Some(slot_delay <= max_re_org_slot_delay(self.spec.seconds_per_slot)) }) .unwrap_or(false) } else { false }; - - // Only attempt single slot re-orgs, and not at epoch boundaries. - let block_slot_ok = parent_node.slot + 1 == head_node.slot - && re_org_block_slot % T::EthSpec::slots_per_epoch() != 0; - - // Only attempt re-orgs with competitive FFG information. - let ffg_competitive = parent_node.unrealized_justified_checkpoint - == head_node.unrealized_justified_checkpoint - && parent_node.unrealized_finalized_checkpoint - == head_node.unrealized_finalized_checkpoint; + if !current_slot_ok { + return Err(DoNotReOrg::HeadDistance.into()); + } // Only attempt a re-org if we have a proposer registered for the re-org slot. - let proposing_at_re_org_slot = block_slot_ok - .then(|| { - let shuffling_decision_root = - parent_node.next_epoch_shuffling_id.shuffling_decision_block; - let proposer_index = self - .beacon_proposer_cache - .lock() - .get_slot::(shuffling_decision_root, re_org_block_slot) - .or_else(|| { - debug!( - self.log, - "Fork choice override proposer shuffling miss"; - "slot" => re_org_block_slot, - "decision_root" => ?shuffling_decision_root, - ); - None - })? - .index; - self.execution_layer - .as_ref() - .map(|el| el.has_proposer_preparation_data_blocking(proposer_index as u64)) - }) - .flatten() - .unwrap_or(false); + let proposing_at_re_org_slot = { + let shuffling_decision_root = info + .parent_node + .next_epoch_shuffling_id + .shuffling_decision_block; + let proposer_index = self + .beacon_proposer_cache + .lock() + .get_slot::(shuffling_decision_root, re_org_block_slot) + .ok_or_else(|| { + debug!( + self.log, + "Fork choice override proposer shuffling miss"; + "slot" => re_org_block_slot, + "decision_root" => ?shuffling_decision_root, + ); + DoNotReOrg::NotProposing + })? + .index as u64; + + self.execution_layer + .as_ref() + .ok_or(ProposerHeadError::Error(Error::ExecutionLayerMissing))? + .has_proposer_preparation_data_blocking(proposer_index) + }; + if !proposing_at_re_org_slot { + return Err(DoNotReOrg::NotProposing.into()); + } // Check that the parent's weight is greater than the participation threshold. // If we are still in the slot of the canonical head block then only check against // a 1x threshold as all attestations may not have arrived yet. - let participation_multiplier = fork_choice_slot.saturating_sub(parent_node.slot).as_u64(); - let participation_ok = participation_threshold_weight.map_or(false, |threshold| { - parent_node.weight >= threshold.saturating_mul(participation_multiplier) - }); + let participation_multiplier = fork_choice_slot + .saturating_sub(info.parent_node.slot) + .as_u64(); + let participation_weight_threshold = info + .participation_weight_threshold + .saturating_mul(participation_multiplier); + let participation_ok = info.parent_node.weight >= participation_weight_threshold; + if !participation_ok { + return Err(DoNotReOrg::ParticipationTooLow { + parent_weight: info.parent_node.weight, + participation_weight_threshold, + } + .into()); + } // If the current slot is already equal to the proposal slot (or we are in the tail end of // the prior slot), then check the actual weight of the head against the re-org threshold. let head_weak = if fork_choice_slot == re_org_block_slot { - re_org_threshold_weight.map_or(false, |threshold| head_node.weight < threshold) + info.head_node.weight < info.re_org_weight_threshold } else { true }; + if !head_weak { + return Err(DoNotReOrg::HeadNotWeak { + head_weight: info.head_node.weight, + re_org_weight_threshold: info.re_org_weight_threshold, + } + .into()); + } // Check that the head block arrived late and is vulnerable to a re-org. This check is only // a heuristic compared to the proper weight check in `get_state_for_re_org`, the reason @@ -3624,37 +3638,27 @@ impl BeaconChain { // attestations for it. We also can't dequeue attestations for the block during the // current slot, which would be necessary for determining its weight. let head_block_late = - self.block_observed_after_attestation_deadline(head_block_root, head_node.slot); - - let might_re_org = current_slot_ok - && block_slot_ok - && ffg_competitive - && proposing_at_re_org_slot - && participation_ok - && head_weak - && head_block_late; - - if !might_re_org { - return Ok(canonical_forkchoice_params); + self.block_observed_after_attestation_deadline(head_block_root, head_slot); + if !head_block_late { + return Err(DoNotReOrg::HeadNotLate.into()); } - let parent_head_hash = parent_node.execution_status.block_hash(); + let parent_head_hash = info.parent_node.execution_status.block_hash(); + let forkchoice_update_params = ForkchoiceUpdateParameters { + head_root: info.parent_node.root, + head_hash: parent_head_hash, + justified_hash: canonical_forkchoice_params.justified_hash, + finalized_hash: canonical_forkchoice_params.finalized_hash, + }; debug!( self.log, "Fork choice update overridden"; - "canonical_head" => ?head_node.root, - "override" => ?parent_node.root, + "canonical_head" => ?head_block_root, + "override" => ?info.parent_node.root, "slot" => fork_choice_slot, ); - let forkchoice_update_params = ForkchoiceUpdateParameters { - head_root: parent_node.root, - head_hash: parent_head_hash, - justified_hash: canonical_forkchoice_params.justified_hash, - finalized_hash: canonical_forkchoice_params.finalized_hash, - }; - Ok(forkchoice_update_params) } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index d69082c6881..17f58b223f4 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -204,7 +204,7 @@ pub enum BeaconChainError { MissingPersistedForkChoice, CommitteePromiseFailed(oneshot_broadcast::Error), MaxCommitteePromises(usize), - SuppressForkChoiceError, + ProposerHeadForkChoiceError(fork_choice::Error), } easy_from_to!(SlotProcessingError, BeaconChainError); diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 4278d8f0b6f..92d4941f8ef 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -1,7 +1,7 @@ use crate::{ForkChoiceStore, InvalidationOperation}; use proto_array::{ - calculate_committee_fraction, Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, - ParticipationThreshold, ProposerHead, ProtoArrayForkChoice, ReOrgThreshold, + Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, ParticipationThreshold, + ProposerHeadError, ProposerHeadInfo, ProtoArrayForkChoice, ReOrgThreshold, }; use slog::{crit, debug, warn, Logger}; use ssz_derive::{Decode, Encode}; @@ -23,7 +23,8 @@ pub enum Error { InvalidAttestation(InvalidAttestation), InvalidAttesterSlashing(AttesterSlashingValidationError), InvalidBlock(InvalidBlock), - ProtoArrayError(String), + ProtoArrayStringError(String), + ProtoArrayError(proto_array::Error), InvalidProtoArrayBytes(String), InvalidLegacyProtoArrayBytes(String), FailedToProcessInvalidExecutionPayload(String), @@ -45,6 +46,7 @@ pub enum Error { ForkChoiceStoreError(T), UnableToSetJustifiedCheckpoint(T), AfterBlockFailed(T), + ProposerHeadError(T), InvalidAnchor { block_slot: Slot, state_slot: Slot, @@ -161,6 +163,12 @@ pub enum InvalidAttestation { impl From for Error { fn from(e: String) -> Self { + Error::ProtoArrayStringError(e) + } +} + +impl From for Error { + fn from(e: proto_array::Error) -> Self { Error::ProtoArrayError(e) } } @@ -568,46 +576,57 @@ where canonical_head: Hash256, re_org_threshold: ReOrgThreshold, participation_threshold: ParticipationThreshold, - ) -> Result> { + ) -> Result>> { // Ensure that fork choice has already been updated for the current slot. This prevents // us from having to take a write lock or do any dequeueing of attestations in this // function. let fc_store_slot = self.fc_store.get_current_slot(); if current_slot != fc_store_slot { - return Err(Error::WrongSlotForGetProposerHead { - current_slot, - fc_store_slot, - }); + return Err(ProposerHeadError::Error( + Error::WrongSlotForGetProposerHead { + current_slot, + fc_store_slot, + }, + )); } // Similarly, the proposer boost for the previous head should already have expired. let proposer_boost_root = self.fc_store.proposer_boost_root(); if !proposer_boost_root.is_zero() { - return Err(Error::ProposerBoostNotExpiredForGetProposerHead { - proposer_boost_root, - }); + return Err(ProposerHeadError::Error( + Error::ProposerBoostNotExpiredForGetProposerHead { + proposer_boost_root, + }, + )); } self.proto_array .get_proposer_head::( current_slot, - self.fc_store.justified_balances(), canonical_head, + self.fc_store.justified_balances(), re_org_threshold, participation_threshold, ) - .map_err(Into::into) + .map_err(ProposerHeadError::convert_inner_error) } - /// Compute the weight corresponding to `committee_percent`. - /// - /// This is a fraction of a single committee weight, measured approximately against - /// the justified balances, just like proposer boost. - pub fn calculate_committee_fraction(&self, committee_percent: u64) -> Option - where - E: EthSpec, - { - calculate_committee_fraction::(self.fc_store.justified_balances(), committee_percent) + pub fn get_preliminary_proposer_head( + &self, + canonical_head: Hash256, + re_org_threshold: ReOrgThreshold, + participation_threshold: ParticipationThreshold, + ) -> Result>> { + let current_slot = self.fc_store.get_current_slot(); + self.proto_array + .get_proposer_head_info::( + current_slot, + canonical_head, + self.fc_store.justified_balances(), + re_org_threshold, + participation_threshold, + ) + .map_err(ProposerHeadError::convert_inner_error) } /// Return information about: diff --git a/consensus/proto_array/src/error.rs b/consensus/proto_array/src/error.rs index e58c8faf299..0a7182ba818 100644 --- a/consensus/proto_array/src/error.rs +++ b/consensus/proto_array/src/error.rs @@ -16,6 +16,8 @@ pub enum Error { InvalidNodeDelta(usize), DeltaOverflow(usize), ProposerBoostOverflow(usize), + ReOrgThresholdOverflow, + ReOrgParticipationThresholdOverflow, IndexOverflow(&'static str), InvalidExecutionDeltaOverflow(usize), InvalidDeltaLen { diff --git a/consensus/proto_array/src/lib.rs b/consensus/proto_array/src/lib.rs index e3520322698..0b8b1cb5fc2 100644 --- a/consensus/proto_array/src/lib.rs +++ b/consensus/proto_array/src/lib.rs @@ -10,8 +10,8 @@ pub use crate::proto_array::{ calculate_committee_fraction, CountUnrealizedFull, InvalidationOperation, }; pub use crate::proto_array_fork_choice::{ - Block, ExecutionStatus, ParticipationThreshold, ProposerHead, ProtoArrayForkChoice, - ReOrgThreshold, + Block, DoNotReOrg, ExecutionStatus, ParticipationThreshold, ProposerHeadError, + ProposerHeadInfo, ProtoArrayForkChoice, ReOrgThreshold, }; pub use error::Error; diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index 145cab17f83..fdcd78483cf 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -174,22 +174,119 @@ where } /// Information about the proposer head used for opportunistic re-orgs. -#[derive(Default, Clone)] -pub struct ProposerHead { - /// If set, the head block that the proposer should build upon. - pub re_org_head: Option, - /// The weight difference between the canonical head and its parent. - pub canonical_head_weight: Option, +#[derive(Clone)] +pub struct ProposerHeadInfo { + /// Information about the *current* head block, which may be re-orged. + pub head_node: ProtoNode, + /// Information about the parent of the current head, which should be selected as the parent + /// for a new proposal *if* a re-org is decided on. + pub parent_node: ProtoNode, /// The computed fraction of the active committee balance below which we can re-org. - pub re_org_weight_threshold: Option, - /// Is this is a single slot re-org? - pub is_single_slot_re_org: bool, - /// Is the proposer head's FFG information competitive with the head to be re-orged? - pub ffg_competitive: bool, - /// Is the re-org block off an epoch boundary where the proposer shuffling could change? - pub shuffling_stable: bool, - /// Is the chain's participation level sufficiently healthy to justify a re-org? - pub participation_ok: bool, + pub re_org_weight_threshold: u64, + /// The computed fraction of the + /// + /// This value requires an additional 1-2x multiplier depending on the current slot. + pub participation_weight_threshold: u64, + /// The current slot from fork choice's point of view, may lead the wall-clock slot by upto + /// 500ms. + pub current_slot: Slot, +} + +/// Error type to enable short-circuiting checks in `get_proposer_head`. +/// +/// This type intentionally does not implement `Debug` so that callers are forced to handle the +/// enum. +#[derive(Clone, PartialEq)] +pub enum ProposerHeadError { + DoNotReOrg(DoNotReOrg), + Error(E), +} + +impl From for ProposerHeadError { + fn from(e: DoNotReOrg) -> ProposerHeadError { + Self::DoNotReOrg(e) + } +} + +impl From for ProposerHeadError { + fn from(e: Error) -> Self { + Self::Error(e) + } +} + +impl ProposerHeadError { + pub fn convert_inner_error(self) -> ProposerHeadError + where + E2: From, + { + self.map_inner_error(E2::from) + } + + pub fn map_inner_error(self, f: impl FnOnce(E1) -> E2) -> ProposerHeadError { + match self { + ProposerHeadError::DoNotReOrg(reason) => ProposerHeadError::DoNotReOrg(reason), + ProposerHeadError::Error(error) => ProposerHeadError::Error(f(error)), + } + } +} + +/// Reasons why a re-org should not be attempted. +/// +/// This type intentionally does not implement `Debug` so that the `Display` impl must be used. +#[derive(Clone, PartialEq)] +pub enum DoNotReOrg { + MissingHeadOrParentNode, + ParentDistance, + HeadDistance, + ShufflingUnstable, + JustificationAndFinalizationNotCompetitive, + ParticipationTooLow { + parent_weight: u64, + participation_weight_threshold: u64, + }, + HeadNotWeak { + head_weight: u64, + re_org_weight_threshold: u64, + }, + HeadNotLate, + NotProposing, + ReOrgsDisabled, +} + +impl std::fmt::Display for DoNotReOrg { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::MissingHeadOrParentNode => write!(f, "unknown head or parent"), + Self::ParentDistance => write!(f, "parent too far from head"), + Self::HeadDistance => write!(f, "head too far from current slot"), + Self::ShufflingUnstable => write!(f, "shuffling unstable at epoch boundary"), + Self::JustificationAndFinalizationNotCompetitive => { + write!(f, "justification or finalization not competitive") + } + Self::ParticipationTooLow { + parent_weight, + participation_weight_threshold, + } => write!( + f, + "participation too low ({parent_weight}/{participation_weight_threshold})" + ), + Self::HeadNotWeak { + head_weight, + re_org_weight_threshold, + } => { + write!(f, "head not weak ({head_weight}/{re_org_weight_threshold})") + } + Self::HeadNotLate => { + write!(f, "head arrived on time") + } + Self::NotProposing => { + write!(f, "not proposing at next slot") + } + Self::ReOrgsDisabled => { + write!(f, "re-orgs disabled in config") + } + } + } } /// New-type for the re-org threshold percentage. @@ -197,7 +294,7 @@ pub struct ProposerHead { #[serde(transparent)] pub struct ReOrgThreshold(pub u64); -/// New-type for the participation threshold percentage. +/// New-type for the re-org participation threshold percentage. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(transparent)] pub struct ParticipationThreshold(pub u64); @@ -351,76 +448,124 @@ impl ProtoArrayForkChoice { .map_err(|e| format!("find_head failed: {:?}", e)) } + /// Get the block to propose on during `current_slot`. + /// + /// This function returns a *definitive* result which should be acted on. pub fn get_proposer_head( &self, current_slot: Slot, + canonical_head: Hash256, justified_balances: &JustifiedBalances, + re_org_threshold: ReOrgThreshold, + participation_threshold: ParticipationThreshold, + ) -> Result> { + let info = self.get_proposer_head_info::( + current_slot, + canonical_head, + justified_balances, + re_org_threshold, + participation_threshold, + )?; + + // Only re-org a single slot. This prevents cascading failures during asynchrony. + let head_slot_ok = info.head_node.slot + 1 == current_slot; + if !head_slot_ok { + return Err(DoNotReOrg::HeadDistance.into()); + } + + // To prevent excessive re-orgs when the chain is struggling, only re-org when participation + // is above the configured threshold. + let parent_weight = info.parent_node.weight; + let participation_weight_threshold = info.participation_weight_threshold.saturating_mul(2); + let participation_ok = parent_weight >= participation_weight_threshold; + if !participation_ok { + return Err(DoNotReOrg::ParticipationTooLow { + parent_weight, + participation_weight_threshold, + } + .into()); + } + + // Only re-org if the head's weight is less than the configured committee fraction. + let head_weight = info.head_node.weight; + let re_org_weight_threshold = info.re_org_weight_threshold; + let weak_head = head_weight < re_org_weight_threshold; + if !weak_head { + return Err(DoNotReOrg::HeadNotWeak { + head_weight, + re_org_weight_threshold, + } + .into()); + } + + // All checks have passed, build upon the parent to re-org the head. + Ok(info) + } + + /// Get information about the block to propose on during `current_slot`. + /// + /// This function returns a *partial* result which must be processed further. + pub fn get_proposer_head_info( + &self, + current_slot: Slot, canonical_head: Hash256, + justified_balances: &JustifiedBalances, re_org_threshold: ReOrgThreshold, participation_threshold: ParticipationThreshold, - ) -> Result { - let nodes = self + ) -> Result> { + let mut nodes = self .proto_array .iter_nodes(&canonical_head) .take(2) + .cloned() .collect::>(); - if nodes.len() != 2 { - return Ok(ProposerHead::default()); + + let parent_node = nodes.pop().ok_or(DoNotReOrg::MissingHeadOrParentNode)?; + let head_node = nodes.pop().ok_or(DoNotReOrg::MissingHeadOrParentNode)?; + + let parent_slot = parent_node.slot; + let head_slot = head_node.slot; + let re_org_block_slot = head_slot + 1; + + // Check parent distance from head. + // Do not check head distance from current slot, as that condition needs to be + // late-evaluated and is elided when `current_slot == head_slot`. + let parent_slot_ok = parent_slot + 1 == head_slot; + if !parent_slot_ok { + return Err(DoNotReOrg::ParentDistance.into()); } - let head_node = nodes[0]; - let parent_node = nodes[1]; - // Only re-org a single slot. This prevents cascading failures during asynchrony. - let is_single_slot_re_org = - parent_node.slot + 1 == head_node.slot && head_node.slot + 1 == current_slot; - - // Do not re-org on the first slot of an epoch because this is liable to change the - // shuffling and rob us of a proposal entirely. A more sophisticated check could be - // done here, but we're prioritising speed and simplicity over precision. - let shuffling_stable = current_slot % E::slots_per_epoch() != 0; - - // Only re-org if the new head will be competitive with the current head's justification and - // finalization. In lieu of computing new justification and finalization for our re-org - // block that hasn't been created yet, just check if the parent we would build on is - // competitive with the head. + // Check shuffling stability. + let shuffling_stable = re_org_block_slot % E::slots_per_epoch() != 0; + if !shuffling_stable { + return Err(DoNotReOrg::ShufflingUnstable.into()); + } + + // Check FFG. let ffg_competitive = parent_node.unrealized_justified_checkpoint == head_node.unrealized_justified_checkpoint && parent_node.unrealized_finalized_checkpoint == head_node.unrealized_finalized_checkpoint; + if !ffg_competitive { + return Err(DoNotReOrg::JustificationAndFinalizationNotCompetitive.into()); + } - // To prevent excessive re-orgs when the chain is struggling, only re-org when participation - // is above the configured threshold. This should not overflow. - let participation_committee_threshold = - calculate_committee_fraction::(justified_balances, participation_threshold.0) - .ok_or_else(|| { - "overflow calculating committee weight for participation threshold".to_string() - })?; - let participation_ok = - parent_node.weight >= participation_committee_threshold.saturating_mul(2); - - // Only re-org if the head's weight is less than the configured committee fraction. + // Compute re-org weight threshold. let re_org_weight_threshold = - calculate_committee_fraction::(justified_balances, re_org_threshold.0).ok_or_else( - || "overflow calculating committee weight for re-org threshold".to_string(), - )?; - let canonical_head_weight = head_node.weight; - let is_weak_head = canonical_head_weight < re_org_weight_threshold; - - let re_org_head = (is_single_slot_re_org - && shuffling_stable - && ffg_competitive - && participation_ok - && is_weak_head) - .then_some(parent_node.root); - - Ok(ProposerHead { - re_org_head, - canonical_head_weight: Some(canonical_head_weight), - re_org_weight_threshold: Some(re_org_weight_threshold), - is_single_slot_re_org, - ffg_competitive, - shuffling_stable, - participation_ok, + calculate_committee_fraction::(justified_balances, re_org_threshold.0) + .ok_or(Error::ReOrgThresholdOverflow)?; + + // Compute participation threshold. + let participation_weight_threshold = + calculate_committee_fraction::(justified_balances, participation_threshold.0) + .ok_or(Error::ReOrgParticipationThresholdOverflow)?; + + Ok(ProposerHeadInfo { + head_node, + parent_node, + re_org_weight_threshold, + participation_weight_threshold, + current_slot, }) } From 370535050eede6408dcf28a5d5c1f7681376c0c8 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 31 Oct 2022 10:38:34 +1100 Subject: [PATCH 28/37] Fix proto array error handling --- beacon_node/beacon_chain/tests/payload_invalidation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 531fdb63047..ec6d1c5a884 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -385,7 +385,7 @@ impl InvalidPayloadRig { .fork_choice_write_lock() .get_head(self.harness.chain.slot().unwrap(), &self.harness.chain.spec) { - Err(ForkChoiceError::ProtoArrayError(e)) if e.contains(s) => (), + Err(ForkChoiceError::ProtoArrayStringError(e)) if e.contains(s) => (), other => panic!("expected {} error, got {:?}", s, other), }; } @@ -1058,7 +1058,7 @@ async fn invalid_parent() { &rig.harness.chain.spec, CountUnrealized::True, ), - Err(ForkChoiceError::ProtoArrayError(message)) + Err(ForkChoiceError::ProtoArrayStringError(message)) if message.contains(&format!( "{:?}", ProtoArrayError::ParentExecutionStatusIsInvalid { From 68bd325d30d1092f3cd5f750d5780f1588a0e077 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 31 Oct 2022 11:02:33 +1100 Subject: [PATCH 29/37] Add metrics --- beacon_node/beacon_chain/src/beacon_chain.rs | 5 +++++ beacon_node/beacon_chain/src/metrics.rs | 12 ++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index ab904cbe701..f1fb107913e 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3398,6 +3398,8 @@ impl BeaconChain { } // Is the current head weak and appropriate for re-orging? + let proposer_head_timer = + metrics::start_timer(&metrics::BLOCK_PRODUCTION_GET_PROPOSER_HEAD_TIMES); let proposer_head = self .canonical_head .fork_choice_read_lock() @@ -3424,6 +3426,7 @@ impl BeaconChain { } }) .ok()?; + drop(proposer_head_timer); let re_org_parent_block = proposer_head.parent_node.root; // Only attempt a re-org if we hit the snapshot cache. @@ -3591,6 +3594,8 @@ impl BeaconChain { &self, canonical_forkchoice_params: &ForkchoiceUpdateParameters, ) -> Result> { + let _timer = metrics::start_timer(&metrics::FORK_CHOICE_OVERRIDE_FCU_TIMES); + // Never override if proposer re-orgs are disabled. let re_org_threshold = self .config diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index ead4a540254..81f491a2bc8 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -94,6 +94,11 @@ lazy_static! { "beacon_block_production_fork_choice_seconds", "Time taken to run fork choice before block production" ); + pub static ref BLOCK_PRODUCTION_GET_PROPOSER_HEAD_TIMES: Result = try_create_histogram_with_buckets( + "beacon_block_production_get_proposer_head_times", + "Time taken for fork choice to compute the proposer head before block production", + exponential_buckets(1e-3, 2.0, 8) + ); pub static ref BLOCK_PRODUCTION_STATE_LOAD_TIMES: Result = try_create_histogram( "beacon_block_production_state_load_seconds", "Time taken to load the base state for block production" @@ -319,8 +324,11 @@ lazy_static! { ); pub static ref FORK_CHOICE_TIMES: Result = try_create_histogram("beacon_fork_choice_seconds", "Full runtime of fork choice"); - pub static ref FORK_CHOICE_FIND_HEAD_TIMES: Result = - try_create_histogram("beacon_fork_choice_find_head_seconds", "Full runtime of fork choice find_head function"); + pub static ref FORK_CHOICE_OVERRIDE_FCU_TIMES: Result = try_create_histogram_with_buckets( + "beacon_fork_choice_override_fcu_seconds", + "Time taken to compute the optional forkchoiceUpdated override", + exponential_buckets(1e-3, 2.0, 8) + ); pub static ref FORK_CHOICE_PROCESS_BLOCK_TIMES: Result = try_create_histogram( "beacon_fork_choice_process_block_seconds", "Time taken to add a block and all attestations to fork choice" From 2b61626fc8e573df682cb2a3ed311a19dc80d4be Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 31 Oct 2022 11:57:18 +1100 Subject: [PATCH 30/37] More metrics --- beacon_node/beacon_chain/src/canonical_head.rs | 2 ++ beacon_node/beacon_chain/src/metrics.rs | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index fd221b4f845..dd64e02edf7 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -786,6 +786,7 @@ impl BeaconChain { new_cached_head: &CachedHead, new_head_proto_block: ProtoBlock, ) -> Result<(), Error> { + let _timer = metrics::start_timer(&metrics::FORK_CHOICE_AFTER_NEW_HEAD_TIMES); let old_snapshot = &old_cached_head.snapshot; let new_snapshot = &new_cached_head.snapshot; let new_head_is_optimistic = new_head_proto_block @@ -923,6 +924,7 @@ impl BeaconChain { new_view: ForkChoiceView, finalized_proto_block: ProtoBlock, ) -> Result<(), Error> { + let _timer = metrics::start_timer(&metrics::FORK_CHOICE_AFTER_FINALIZATION_TIMES); let new_snapshot = &new_cached_head.snapshot; let finalized_block_is_optimistic = finalized_proto_block .execution_status diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 81f491a2bc8..2a7374001cf 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -322,13 +322,26 @@ lazy_static! { "beacon_reorgs_total", "Count of occasions fork choice has switched to a different chain" ); - pub static ref FORK_CHOICE_TIMES: Result = - try_create_histogram("beacon_fork_choice_seconds", "Full runtime of fork choice"); + pub static ref FORK_CHOICE_TIMES: Result = try_create_histogram_with_buckets( + "beacon_fork_choice_seconds", + "Full runtime of fork choice", + linear_buckets(10e-3, 20e-3, 10) + ); pub static ref FORK_CHOICE_OVERRIDE_FCU_TIMES: Result = try_create_histogram_with_buckets( "beacon_fork_choice_override_fcu_seconds", "Time taken to compute the optional forkchoiceUpdated override", exponential_buckets(1e-3, 2.0, 8) ); + pub static ref FORK_CHOICE_AFTER_NEW_HEAD_TIMES: Result = try_create_histogram_with_buckets( + "beacon_fork_choice_after_new_head_seconds", + "Time taken to run `after_new_head`", + exponential_buckets(1e-3, 2.0, 10) + ); + pub static ref FORK_CHOICE_AFTER_FINALIZATION_TIMES: Result = try_create_histogram_with_buckets( + "beacon_fork_choice_after_finalization_seconds", + "Time taken to run `after_finalization`", + exponential_buckets(1e-3, 2.0, 10) + ); pub static ref FORK_CHOICE_PROCESS_BLOCK_TIMES: Result = try_create_histogram( "beacon_fork_choice_process_block_seconds", "Time taken to add a block and all attestations to fork choice" From bec0f5964496c6336746d36a176a5dc3a549cef7 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 31 Oct 2022 12:50:54 +1100 Subject: [PATCH 31/37] Metric for block proc fork choice --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 ++ beacon_node/beacon_chain/src/metrics.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index f1fb107913e..b81a78512a6 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2873,6 +2873,7 @@ impl BeaconChain { if !payload_verification_status.is_optimistic() && block.slot() + EARLY_ATTESTER_CACHE_HISTORIC_SLOTS >= current_slot { + let fork_choice_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_FORK_CHOICE); match fork_choice.get_head(current_slot, &self.spec) { // This block became the head, add it to the early attester cache. Ok(new_head_root) if new_head_root == block_root => { @@ -2906,6 +2907,7 @@ impl BeaconChain { "error" => ?e ), } + drop(fork_choice_timer); } // Register sync aggregate with validator monitor diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 2a7374001cf..d596d4241c5 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -72,6 +72,11 @@ lazy_static! { "beacon_block_processing_attestation_observation_seconds", "Time spent hashing and remembering all the attestations in the block" ); + pub static ref BLOCK_PROCESSING_FORK_CHOICE: Result = try_create_histogram_with_buckets( + "beacon_block_processing_fork_choice_seconds", + "Time spent running fork choice's `get_head` during block import", + exponential_buckets(1e-3, 2.0, 8) + ); pub static ref BLOCK_SYNC_AGGREGATE_SET_BITS: Result = try_create_int_gauge( "block_sync_aggregate_set_bits", "The number of true bits in the last sync aggregate in a block" From e0f8a2c0f6e64e6a0dca181828ab6d05046a63f3 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 31 Oct 2022 15:33:53 +1100 Subject: [PATCH 32/37] Add docs --- book/src/SUMMARY.md | 1 + book/src/late-block-re-orgs.md | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 book/src/late-block-re-orgs.md diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index d05677465b5..8657918c354 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -46,6 +46,7 @@ * [Pre-Releases](./advanced-pre-releases.md) * [Release Candidates](./advanced-release-candidates.md) * [MEV and Lighthouse](./builders.md) + * [Late Block Re-orgs](./late-block-re-orgs.md) * [Contributing](./contributing.md) * [Development Environment](./setup.md) * [FAQs](./faq.md) diff --git a/book/src/late-block-re-orgs.md b/book/src/late-block-re-orgs.md new file mode 100644 index 00000000000..d067d28e31d --- /dev/null +++ b/book/src/late-block-re-orgs.md @@ -0,0 +1,59 @@ +# Late Block Re-orgs + +Since v3.3.0 Lighthouse will opportunistically re-org late blocks when proposing. + +This feature is intended to disincentivise late blocks and improve network health. Proposing a +re-orging block is also more profitable for the proposer because it increases the number of +attestations and transactions that can be included. + +## Command line flags + +There are three flags which control the re-orging behaviour: + +* `--disable-proposer-reorgs`: turn re-orging off (it's on by default). +* `--proposer-reorg-threshold N`: attempt to orphan blocks with less than N% of the committee vote. If this parameter isn't set then N defaults to 20% when the feature is enabled. +* `--proposer-reorg-participation-threshold N`: only attempt to re-org late blocks when the on-chain participation is approximately N% or greater. Default is 70%. + +All flags should be applied to `lighthouse bn`. The default configuration is recommended as it +balances the chance of the re-org succeeding against the chance of failure due to attestations +arriving late and making the re-org block non-viable. + +## Safeguards + +To prevent excessive re-orgs there are several safeguards in place that limit when a re-org +will be attempted. + +The full conditions are described in [the spec][] but the most important ones are: + +* Only single-slot re-orgs: Lighthouse will build a block at N + 1 to re-org N by building on the + parent N - 1. The result is a chain with exactly one skipped slot. +* No epoch boundaries: to ensure that the selected proposer does not change, Lighthouse will + not propose a re-orging block in the 0th slot of an epoch. + +## Logs + +You can track the reasons for re-orgs being attempted (or not) via Lighthouse's logs. + +A pair of messages at `INFO` level will be logged if a re-org opportunity is detected: + +> INFO Attempting re-org due to weak head threshold_weight: 45455983852725, head_weight: 0, parent: 0x09d953b69041f280758400c671130d174113bbf57c2d26553a77fb514cad4890, weak_head: 0xf64f8e5ed617dc18c1e759dab5d008369767c3678416dac2fe1d389562842b49 + +> INFO Proposing block to re-org current head head_to_reorg: 0xf64f…2b49, slot: 1105320 + +This should be followed shortly after by a `WARN` log indicating that a re-org occurred. This is +expected and normal: + +> WARN Beacon chain re-org reorg_distance: 1, new_slot: 1105320, new_head: 0x72791549e4ca792f91053bc7cf1e55c6fbe745f78ce7a16fc3acb6f09161becd, previous_slot: 1105319, previous_head: 0xf64f8e5ed617dc18c1e759dab5d008369767c3678416dac2fe1d389562842b49 + +In case a re-org is not viable (which should be most of the time), Lighthouse will just propose a +block as normal and log the reason the re-org was not attempted at debug level: + +> DEBG Not attempting re-org reason: head not late + +If you are interested in digging into the timing of `forkchoiceUpdated` messages sent to the +execution layer, there is also a debug log for the suppression of `forkchoiceUpdated` messages +when Lighthouse thinks that a re-org is likely: + +> DEBG Fork choice update overridden slot: 1105320, override: 0x09d953b69041f280758400c671130d174113bbf57c2d26553a77fb514cad4890, canonical_head: 0xf64f8e5ed617dc18c1e759dab5d008369767c3678416dac2fe1d389562842b49 + +[the spec]: https://github.com/ethereum/consensus-specs/pull/3034 From aa0d85ee6672c9f02d616947234e328293dd5d86 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 10 Nov 2022 14:32:54 +1100 Subject: [PATCH 33/37] Update some comments --- consensus/fork_choice/src/fork_choice.rs | 4 ++++ consensus/proto_array/src/proto_array_fork_choice.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 92d4941f8ef..8a2722d0953 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -570,6 +570,10 @@ where Ok(head_root) } + /// Get the block to build on as proposer, taking into account proposer re-orgs. + /// + /// You *must* call `get_head` for the proposal slot prior to calling this function and pass + /// in the result of `get_head` as `canonical_head`. pub fn get_proposer_head( &self, current_slot: Slot, diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index fdcd78483cf..fbc0d63871c 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -183,7 +183,7 @@ pub struct ProposerHeadInfo { pub parent_node: ProtoNode, /// The computed fraction of the active committee balance below which we can re-org. pub re_org_weight_threshold: u64, - /// The computed fraction of the + /// The computed fraction of the active committee balance which participation must exceed. /// /// This value requires an additional 1-2x multiplier depending on the current slot. pub participation_weight_threshold: u64, From 7b81b2d9e5e4d0ad1be4e5a8b1dc21c265037d6e Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 28 Nov 2022 17:56:29 +1100 Subject: [PATCH 34/37] Finality distance circuit-breaker --- beacon_node/beacon_chain/src/beacon_chain.rs | 22 +------ beacon_node/beacon_chain/src/builder.rs | 14 ++--- beacon_node/beacon_chain/src/chain_config.rs | 13 ++-- .../http_api/tests/interactive_tests.rs | 35 +++-------- beacon_node/src/cli.rs | 10 +-- beacon_node/src/config.rs | 11 ++-- book/src/late-block-re-orgs.md | 3 +- consensus/fork_choice/src/fork_choice.rs | 12 ++-- consensus/proto_array/src/error.rs | 1 - consensus/proto_array/src/lib.rs | 4 +- .../src/proto_array_fork_choice.rs | 62 +++++++------------ lighthouse/tests/beacon_node.rs | 17 +++-- 12 files changed, 80 insertions(+), 124 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a2f9c0d4785..9b62c393c08 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3449,7 +3449,7 @@ impl BeaconChain { slot, canonical_head, re_org_threshold, - self.config.re_org_participation_threshold, + self.config.re_org_max_epochs_since_finalization, ) .map_err(|e| match e { ProposerHeadError::DoNotReOrg(reason) => { @@ -3653,7 +3653,7 @@ impl BeaconChain { .get_preliminary_proposer_head( head_block_root, re_org_threshold, - self.config.re_org_participation_threshold, + self.config.re_org_max_epochs_since_finalization, ) .map_err(|e| e.map_inner_error(Error::ProposerHeadForkChoiceError))?; @@ -3714,24 +3714,6 @@ impl BeaconChain { return Err(DoNotReOrg::NotProposing.into()); } - // Check that the parent's weight is greater than the participation threshold. - // If we are still in the slot of the canonical head block then only check against - // a 1x threshold as all attestations may not have arrived yet. - let participation_multiplier = fork_choice_slot - .saturating_sub(info.parent_node.slot) - .as_u64(); - let participation_weight_threshold = info - .participation_weight_threshold - .saturating_mul(participation_multiplier); - let participation_ok = info.parent_node.weight >= participation_weight_threshold; - if !participation_ok { - return Err(DoNotReOrg::ParticipationTooLow { - parent_weight: info.parent_node.weight, - participation_weight_threshold, - } - .into()); - } - // If the current slot is already equal to the proposal slot (or we are in the tail end of // the prior slot), then check the actual weight of the head against the re-org threshold. let head_weak = if fork_choice_slot == re_org_block_slot { diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index b607cc2a086..64e2ebabe15 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -22,7 +22,7 @@ use fork_choice::{ForkChoice, ResetPayloadStatuses}; use futures::channel::mpsc::Sender; use operation_pool::{OperationPool, PersistedOperationPool}; use parking_lot::RwLock; -use proto_array::{ParticipationThreshold, ReOrgThreshold}; +use proto_array::ReOrgThreshold; use slasher::Slasher; use slog::{crit, error, info, Logger}; use slot_clock::{SlotClock, TestingSlotClock}; @@ -32,8 +32,8 @@ use std::time::Duration; use store::{Error as StoreError, HotColdDB, ItemStore, KeyValueStoreOp}; use task_executor::{ShutdownReason, TaskExecutor}; use types::{ - BeaconBlock, BeaconState, ChainSpec, Checkpoint, EthSpec, Graffiti, Hash256, PublicKeyBytes, - Signature, SignedBeaconBlock, Slot, + BeaconBlock, BeaconState, ChainSpec, Checkpoint, Epoch, EthSpec, Graffiti, Hash256, + PublicKeyBytes, Signature, SignedBeaconBlock, Slot, }; /// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing @@ -166,12 +166,12 @@ where self } - /// Sets the proposer re-org participation threshold. - pub fn proposer_re_org_participation_threshold( + /// Sets the proposer re-org max epochs since finalization. + pub fn proposer_re_org_max_epochs_since_finalization( mut self, - threshold: ParticipationThreshold, + epochs_since_finalization: Epoch, ) -> Self { - self.chain_config.re_org_participation_threshold = threshold; + self.chain_config.re_org_max_epochs_since_finalization = epochs_since_finalization; self } diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index 84fcfddf67c..c4c6966732d 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -1,11 +1,10 @@ -pub use proto_array::{CountUnrealizedFull, ParticipationThreshold, ReOrgThreshold}; +pub use proto_array::{CountUnrealizedFull, ReOrgThreshold}; use serde_derive::{Deserialize, Serialize}; use std::time::Duration; -use types::Checkpoint; +use types::{Checkpoint, Epoch}; pub const DEFAULT_RE_ORG_THRESHOLD: ReOrgThreshold = ReOrgThreshold(20); -pub const DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD: ParticipationThreshold = - ParticipationThreshold(70); +pub const DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION: Epoch = Epoch::new(2); pub const DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT: u64 = 250; /// Default fraction of a slot lookahead for payload preparation (12/3 = 4 seconds on mainnet). @@ -33,8 +32,8 @@ pub struct ChainConfig { pub max_network_size: usize, /// Maximum percentage of committee weight at which to attempt re-orging the canonical head. pub re_org_threshold: Option, - /// Minimum participation at which a proposer re-org should be attempted. - pub re_org_participation_threshold: ParticipationThreshold, + /// Maximum number of epochs since finalization for attempting a proposer re-org. + pub re_org_max_epochs_since_finalization: Epoch, /// Number of milliseconds to wait for fork choice before proposing a block. /// /// If set to 0 then block proposal will not wait for fork choice at all. @@ -77,7 +76,7 @@ impl Default for ChainConfig { enable_lock_timeouts: true, max_network_size: 10 * 1_048_576, // 10M re_org_threshold: Some(DEFAULT_RE_ORG_THRESHOLD), - re_org_participation_threshold: DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD, + re_org_max_epochs_since_finalization: DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, fork_choice_before_proposal_timeout_ms: DEFAULT_FORK_CHOICE_BEFORE_PROPOSAL_TIMEOUT, // Builder fallback configs that are set in `clap` will override these. builder_fallback_skips: 3, diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index baa036ce2a1..e8434ea1a6d 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -1,7 +1,7 @@ //! Generic tests that make use of the (newer) `InteractiveApiTester` use crate::common::*; use beacon_chain::{ - chain_config::{ParticipationThreshold, ReOrgThreshold}, + chain_config::ReOrgThreshold, test_utils::{AttestationStrategy, BlockStrategy}, }; use eth2::types::DepositContractData; @@ -14,8 +14,8 @@ use std::sync::Arc; use std::time::Duration; use tree_hash::TreeHash; use types::{ - Address, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, MainnetEthSpec, - ProposerPreparationData, Slot, + Address, Epoch, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, + MainnetEthSpec, ProposerPreparationData, Slot, }; type E = MainnetEthSpec; @@ -100,7 +100,7 @@ pub struct ReOrgTest { /// Number of slots between head block and block proposal slot. head_distance: u64, re_org_threshold: u64, - participation_threshold: u64, + max_epochs_since_finalization: u64, percent_parent_votes: usize, percent_empty_votes: usize, percent_head_votes: usize, @@ -116,7 +116,7 @@ impl Default for ReOrgTest { parent_distance: 1, head_distance: 1, re_org_threshold: 20, - participation_threshold: 70, + max_epochs_since_finalization: 2, percent_parent_votes: 100, percent_empty_votes: 100, percent_head_votes: 0, @@ -153,6 +153,7 @@ pub async fn proposer_boost_re_org_bad_ffg() { .await; } +/* FIXME(sproul): write finality test #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub async fn proposer_boost_re_org_low_total_participation() { proposer_boost_re_org_test(ReOrgTest { @@ -166,6 +167,7 @@ pub async fn proposer_boost_re_org_low_total_participation() { }) .await; } +*/ #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub async fn proposer_boost_re_org_parent_distance() { @@ -204,23 +206,6 @@ pub async fn proposer_boost_re_org_very_unhealthy() { .await; } -/// Participation initially appears high at 80%, but drops off to 80+50/2 = 65% after the head -/// block. This results in a misprediction. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -pub async fn proposer_boost_re_org_participation_misprediction() { - proposer_boost_re_org_test(ReOrgTest { - head_slot: Slot::new(30), - percent_parent_votes: 80, - percent_empty_votes: 50, - percent_head_votes: 0, - participation_threshold: 70, - should_re_org: false, - misprediction: true, - ..Default::default() - }) - .await; -} - /// The head block is late but still receives 30% of the committee vote, leading to a misprediction. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub async fn proposer_boost_re_org_weight_misprediction() { @@ -248,7 +233,7 @@ pub async fn proposer_boost_re_org_test( parent_distance, head_distance, re_org_threshold, - participation_threshold, + max_epochs_since_finalization, percent_parent_votes, percent_empty_votes, percent_head_votes, @@ -284,8 +269,8 @@ pub async fn proposer_boost_re_org_test( Some(Box::new(move |builder| { builder .proposer_re_org_threshold(Some(ReOrgThreshold(re_org_threshold))) - .proposer_re_org_participation_threshold(ParticipationThreshold( - participation_threshold, + .proposer_re_org_max_epochs_since_finalization(Epoch::new( + max_epochs_since_finalization, )) })), ) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 421c7127875..c5f4cc8adf4 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -776,11 +776,11 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .conflicts_with("disable-proposer-reorgs") ) .arg( - Arg::with_name("proposer-reorg-participation-threshold") - .long("proposer-reorg-participation-threshold") - .value_name("PERCENT") - .help("Minimum participation percentage at proposer reorgs are allowed. \ - Default: 70%") + Arg::with_name("proposer-reorg-epochs-since-finalization") + .long("proposer-reorg-epochs-since-finalization") + .value_name("EPOCHS") + .help("Maximum number of epochs since finalization at which proposer reorgs are \ + allowed. Default: 2") .conflicts_with("disable-proposer-reorgs") ) .arg( diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index ce9643d988f..2dbe95a61a8 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1,6 +1,6 @@ use beacon_chain::chain_config::{ - ParticipationThreshold, ReOrgThreshold, DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR, - DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD, DEFAULT_RE_ORG_THRESHOLD, + ReOrgThreshold, DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR, + DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, DEFAULT_RE_ORG_THRESHOLD, }; use clap::ArgMatches; use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG; @@ -686,10 +686,9 @@ pub fn get_config( .map(ReOrgThreshold) .unwrap_or(DEFAULT_RE_ORG_THRESHOLD), ); - client_config.chain.re_org_participation_threshold = - clap_utils::parse_optional(cli_args, "proposer-reorg-participation-threshold")? - .map(ParticipationThreshold) - .unwrap_or(DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD); + client_config.chain.re_org_max_epochs_since_finalization = + clap_utils::parse_optional(cli_args, "proposer-reorg-epochs-since-finalization")? + .unwrap_or(DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION); } // Note: This overrides any previous flags that enable this option. diff --git a/book/src/late-block-re-orgs.md b/book/src/late-block-re-orgs.md index d067d28e31d..8175b5d12cc 100644 --- a/book/src/late-block-re-orgs.md +++ b/book/src/late-block-re-orgs.md @@ -12,7 +12,8 @@ There are three flags which control the re-orging behaviour: * `--disable-proposer-reorgs`: turn re-orging off (it's on by default). * `--proposer-reorg-threshold N`: attempt to orphan blocks with less than N% of the committee vote. If this parameter isn't set then N defaults to 20% when the feature is enabled. -* `--proposer-reorg-participation-threshold N`: only attempt to re-org late blocks when the on-chain participation is approximately N% or greater. Default is 70%. +* `--proposer-reorg-epochs-since-finalization N`: only attempt to re-org late blocks when the number of epochs since finalization is less than or equal to N. The default is 2 epochs, + meaning re-orgs will only be attempted when the chain is finalizing optimally. All flags should be applied to `lighthouse bn`. The default configuration is recommended as it balances the chance of the re-org succeeding against the chance of failure due to attestations diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 8a2722d0953..290cef78ab5 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -1,7 +1,7 @@ use crate::{ForkChoiceStore, InvalidationOperation}; use proto_array::{ - Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, ParticipationThreshold, - ProposerHeadError, ProposerHeadInfo, ProtoArrayForkChoice, ReOrgThreshold, + Block as ProtoBlock, CountUnrealizedFull, ExecutionStatus, ProposerHeadError, ProposerHeadInfo, + ProtoArrayForkChoice, ReOrgThreshold, }; use slog::{crit, debug, warn, Logger}; use ssz_derive::{Decode, Encode}; @@ -579,7 +579,7 @@ where current_slot: Slot, canonical_head: Hash256, re_org_threshold: ReOrgThreshold, - participation_threshold: ParticipationThreshold, + max_epochs_since_finalization: Epoch, ) -> Result>> { // Ensure that fork choice has already been updated for the current slot. This prevents // us from having to take a write lock or do any dequeueing of attestations in this @@ -610,7 +610,7 @@ where canonical_head, self.fc_store.justified_balances(), re_org_threshold, - participation_threshold, + max_epochs_since_finalization, ) .map_err(ProposerHeadError::convert_inner_error) } @@ -619,7 +619,7 @@ where &self, canonical_head: Hash256, re_org_threshold: ReOrgThreshold, - participation_threshold: ParticipationThreshold, + max_epochs_since_finalization: Epoch, ) -> Result>> { let current_slot = self.fc_store.get_current_slot(); self.proto_array @@ -628,7 +628,7 @@ where canonical_head, self.fc_store.justified_balances(), re_org_threshold, - participation_threshold, + max_epochs_since_finalization, ) .map_err(ProposerHeadError::convert_inner_error) } diff --git a/consensus/proto_array/src/error.rs b/consensus/proto_array/src/error.rs index 0a7182ba818..c55739da792 100644 --- a/consensus/proto_array/src/error.rs +++ b/consensus/proto_array/src/error.rs @@ -17,7 +17,6 @@ pub enum Error { DeltaOverflow(usize), ProposerBoostOverflow(usize), ReOrgThresholdOverflow, - ReOrgParticipationThresholdOverflow, IndexOverflow(&'static str), InvalidExecutionDeltaOverflow(usize), InvalidDeltaLen { diff --git a/consensus/proto_array/src/lib.rs b/consensus/proto_array/src/lib.rs index 0b8b1cb5fc2..f2b29e1c7b2 100644 --- a/consensus/proto_array/src/lib.rs +++ b/consensus/proto_array/src/lib.rs @@ -10,8 +10,8 @@ pub use crate::proto_array::{ calculate_committee_fraction, CountUnrealizedFull, InvalidationOperation, }; pub use crate::proto_array_fork_choice::{ - Block, DoNotReOrg, ExecutionStatus, ParticipationThreshold, ProposerHeadError, - ProposerHeadInfo, ProtoArrayForkChoice, ReOrgThreshold, + Block, DoNotReOrg, ExecutionStatus, ProposerHeadError, ProposerHeadInfo, ProtoArrayForkChoice, + ReOrgThreshold, }; pub use error::Error; diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index fbc0d63871c..cbd369ae6ec 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -183,10 +183,6 @@ pub struct ProposerHeadInfo { pub parent_node: ProtoNode, /// The computed fraction of the active committee balance below which we can re-org. pub re_org_weight_threshold: u64, - /// The computed fraction of the active committee balance which participation must exceed. - /// - /// This value requires an additional 1-2x multiplier depending on the current slot. - pub participation_weight_threshold: u64, /// The current slot from fork choice's point of view, may lead the wall-clock slot by upto /// 500ms. pub current_slot: Slot, @@ -236,13 +232,13 @@ impl ProposerHeadError { #[derive(Clone, PartialEq)] pub enum DoNotReOrg { MissingHeadOrParentNode, + MissingHeadFinalizedCheckpoint, ParentDistance, HeadDistance, ShufflingUnstable, JustificationAndFinalizationNotCompetitive, - ParticipationTooLow { - parent_weight: u64, - participation_weight_threshold: u64, + ChainNotFinalizing { + epochs_since_finalization: u64, }, HeadNotWeak { head_weight: u64, @@ -257,18 +253,18 @@ impl std::fmt::Display for DoNotReOrg { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { Self::MissingHeadOrParentNode => write!(f, "unknown head or parent"), + Self::MissingHeadFinalizedCheckpoint => write!(f, "finalized checkpoint missing"), Self::ParentDistance => write!(f, "parent too far from head"), Self::HeadDistance => write!(f, "head too far from current slot"), Self::ShufflingUnstable => write!(f, "shuffling unstable at epoch boundary"), Self::JustificationAndFinalizationNotCompetitive => { write!(f, "justification or finalization not competitive") } - Self::ParticipationTooLow { - parent_weight, - participation_weight_threshold, + Self::ChainNotFinalizing { + epochs_since_finalization, } => write!( f, - "participation too low ({parent_weight}/{participation_weight_threshold})" + "chain not finalizing ({epochs_since_finalization} epochs since finalization)" ), Self::HeadNotWeak { head_weight, @@ -294,11 +290,6 @@ impl std::fmt::Display for DoNotReOrg { #[serde(transparent)] pub struct ReOrgThreshold(pub u64); -/// New-type for the re-org participation threshold percentage. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(transparent)] -pub struct ParticipationThreshold(pub u64); - #[derive(PartialEq)] pub struct ProtoArrayForkChoice { pub(crate) proto_array: ProtoArray, @@ -457,14 +448,14 @@ impl ProtoArrayForkChoice { canonical_head: Hash256, justified_balances: &JustifiedBalances, re_org_threshold: ReOrgThreshold, - participation_threshold: ParticipationThreshold, + max_epochs_since_finalization: Epoch, ) -> Result> { let info = self.get_proposer_head_info::( current_slot, canonical_head, justified_balances, re_org_threshold, - participation_threshold, + max_epochs_since_finalization, )?; // Only re-org a single slot. This prevents cascading failures during asynchrony. @@ -473,19 +464,6 @@ impl ProtoArrayForkChoice { return Err(DoNotReOrg::HeadDistance.into()); } - // To prevent excessive re-orgs when the chain is struggling, only re-org when participation - // is above the configured threshold. - let parent_weight = info.parent_node.weight; - let participation_weight_threshold = info.participation_weight_threshold.saturating_mul(2); - let participation_ok = parent_weight >= participation_weight_threshold; - if !participation_ok { - return Err(DoNotReOrg::ParticipationTooLow { - parent_weight, - participation_weight_threshold, - } - .into()); - } - // Only re-org if the head's weight is less than the configured committee fraction. let head_weight = info.head_node.weight; let re_org_weight_threshold = info.re_org_weight_threshold; @@ -511,7 +489,7 @@ impl ProtoArrayForkChoice { canonical_head: Hash256, justified_balances: &JustifiedBalances, re_org_threshold: ReOrgThreshold, - participation_threshold: ParticipationThreshold, + max_epochs_since_finalization: Epoch, ) -> Result> { let mut nodes = self .proto_array @@ -527,6 +505,20 @@ impl ProtoArrayForkChoice { let head_slot = head_node.slot; let re_org_block_slot = head_slot + 1; + // Check finalization distance. + let proposal_epoch = re_org_block_slot.epoch(E::slots_per_epoch()); + let finalized_epoch = head_node + .unrealized_finalized_checkpoint + .ok_or(DoNotReOrg::MissingHeadFinalizedCheckpoint)? + .epoch; + let epochs_since_finalization = proposal_epoch.saturating_sub(finalized_epoch).as_u64(); + if epochs_since_finalization > max_epochs_since_finalization.as_u64() { + return Err(DoNotReOrg::ChainNotFinalizing { + epochs_since_finalization, + } + .into()); + } + // Check parent distance from head. // Do not check head distance from current slot, as that condition needs to be // late-evaluated and is elided when `current_slot == head_slot`. @@ -555,16 +547,10 @@ impl ProtoArrayForkChoice { calculate_committee_fraction::(justified_balances, re_org_threshold.0) .ok_or(Error::ReOrgThresholdOverflow)?; - // Compute participation threshold. - let participation_weight_threshold = - calculate_committee_fraction::(justified_balances, participation_threshold.0) - .ok_or(Error::ReOrgParticipationThresholdOverflow)?; - Ok(ProposerHeadInfo { head_node, parent_node, re_org_weight_threshold, - participation_weight_threshold, current_slot, }) } diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 5a5aff0cb7c..07c583da5cb 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -2,7 +2,7 @@ use beacon_node::{beacon_chain::CountUnrealizedFull, ClientConfig as Config}; use crate::exec::{CommandLineTestExec, CompletedTest}; use beacon_node::beacon_chain::chain_config::{ - DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD, DEFAULT_RE_ORG_THRESHOLD, + DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, DEFAULT_RE_ORG_THRESHOLD, }; use eth1::Eth1Endpoint; use lighthouse_network::PeerId; @@ -1537,8 +1537,8 @@ fn enable_proposer_re_orgs_default() { Some(DEFAULT_RE_ORG_THRESHOLD) ); assert_eq!( - config.chain.re_org_participation_threshold, - DEFAULT_RE_ORG_PARTICIPATION_THRESHOLD, + config.chain.re_org_max_epochs_since_finalization, + DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, ); }); } @@ -1560,11 +1560,16 @@ fn proposer_re_org_threshold() { } #[test] -fn proposer_re_org_participation_threshold() { +fn proposer_re_org_max_epochs_since_finalization() { CommandLineTest::new() - .flag("proposer-reorg-participation-threshold", Some("20")) + .flag("proposer-reorg-epochs-since-finalization", Some("8")) .run() - .with_config(|config| assert_eq!(config.chain.re_org_participation_threshold.0, 20)); + .with_config(|config| { + assert_eq!( + config.chain.re_org_max_epochs_since_finalization.as_u64(), + 8 + ) + }); } #[test] From 4cb7aa54c030b0da071bfde31e4b81b8af1c84e5 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 9 Dec 2022 16:26:05 +1100 Subject: [PATCH 35/37] Tests for finality/no-finality --- .../http_api/tests/interactive_tests.rs | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index e8434ea1a6d..f41193280f5 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -153,21 +153,27 @@ pub async fn proposer_boost_re_org_bad_ffg() { .await; } -/* FIXME(sproul): write finality test #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -pub async fn proposer_boost_re_org_low_total_participation() { +pub async fn proposer_boost_re_org_no_finality() { proposer_boost_re_org_test(ReOrgTest { - head_slot: Slot::new(30), - percent_parent_votes: 70, - percent_empty_votes: 60, - percent_head_votes: 10, - participation_threshold: 71, + head_slot: Slot::new(96), + percent_parent_votes: 100, + percent_empty_votes: 0, + percent_head_votes: 100, should_re_org: false, ..Default::default() }) .await; } -*/ + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_finality() { + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(129), + ..Default::default() + }) + .await; +} #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub async fn proposer_boost_re_org_parent_distance() { From bc47bb0ac6487508b77e14ca3f99dc2a29a5b2e5 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 12 Dec 2022 11:04:38 +1100 Subject: [PATCH 36/37] Fix bug in proposer shuffling determination --- beacon_node/beacon_chain/src/beacon_chain.rs | 6 +++++- beacon_node/http_api/tests/interactive_tests.rs | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5c8fde29317..f36bc30c52f 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3816,8 +3816,12 @@ impl BeaconChain { // Only attempt a re-org if we have a proposer registered for the re-org slot. let proposing_at_re_org_slot = { + // The proposer shuffling has the same decision root as the next epoch attestation + // shuffling. We know our re-org block is not on the epoch boundary, so it has the + // same proposer shuffling as the head (but not necessarily the parent which may lie + // in the previous epoch). let shuffling_decision_root = info - .parent_node + .head_node .next_epoch_shuffling_id .shuffling_decision_block; let proposer_index = self diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index f41193280f5..17a3624afed 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -143,6 +143,15 @@ pub async fn proposer_boost_re_org_epoch_boundary() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn proposer_boost_re_org_slot_after_epoch_boundary() { + proposer_boost_re_org_test(ReOrgTest { + head_slot: Slot::new(33), + ..Default::default() + }) + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] pub async fn proposer_boost_re_org_bad_ffg() { proposer_boost_re_org_test(ReOrgTest { From 357bc979b05ed281ca0902bf907b297411764620 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 12 Dec 2022 11:25:57 +1100 Subject: [PATCH 37/37] Update book --- book/src/builders.md | 12 ++++++++---- book/src/late-block-re-orgs.md | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/book/src/builders.md b/book/src/builders.md index 99fae5b3e76..f2a4b3936a5 100644 --- a/book/src/builders.md +++ b/book/src/builders.md @@ -200,19 +200,23 @@ for `INFO` and `WARN` messages indicating why the builder was not used. Examples of messages indicating fallback to a locally produced block are: ``` -INFO No payload provided by connected builder. +INFO Builder did not return a payload ``` ``` -WARN Unable to retrieve a payload from a connected builder +WARN Builder error when requesting payload ``` ``` -INFO The value offered by the connected builder does not meet the configured profit threshold. +WARN Builder returned invalid payload ``` ``` -INFO Due to poor chain health the local execution engine will be used for payload construction. +INFO Builder payload ignored +``` + +``` +INFO Chain is unhealthy, using local payload ``` In case of fallback you should see a log indicating that the locally produced payload was diff --git a/book/src/late-block-re-orgs.md b/book/src/late-block-re-orgs.md index 8175b5d12cc..0014af8f152 100644 --- a/book/src/late-block-re-orgs.md +++ b/book/src/late-block-re-orgs.md @@ -1,6 +1,6 @@ # Late Block Re-orgs -Since v3.3.0 Lighthouse will opportunistically re-org late blocks when proposing. +Since v3.4.0 Lighthouse will opportunistically re-org late blocks when proposing. This feature is intended to disincentivise late blocks and improve network health. Proposing a re-orging block is also more profitable for the proposer because it increases the number of