Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix phase0 block reward in rewards API #5101

Merged
merged 15 commits into from
Sep 17, 2024
Merged

Fix phase0 block reward in rewards API #5101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
71ef86b
Added Block Rewards
Gua00va Oct 26, 2023
9bcaa83
added new type
Gua00va Nov 10, 2023
4e9725c
added enum
Gua00va Nov 12, 2023
4ce8f2a
Fix phase0 block reward in rewards API (#4929)
dknopik Jan 20, 2024
4449c54
Merge 'guav00a/proposer-rewards-api'
dknopik Jan 29, 2024
1462dc4
Merge unstable
dknopik Jan 29, 2024
3204172
Revamp phase0 reward API tests
dknopik Jan 31, 2024
d3e2493
Merge branch 'unstable' into fix-4929
dknopik Feb 22, 2024
df16707
Merge branch 'unstable' into fix-4929
dknopik Mar 14, 2024
614465d
Merge remote-tracking branch 'origin/unstable' into fix-4929
michaelsproul Sep 10, 2024
e45de97
Fix merge fallout
michaelsproul Sep 10, 2024
4fd0e50
Remove junk revived in merge
michaelsproul Sep 10, 2024
2007fd4
Address review
dknopik Sep 11, 2024
fec0996
Merge branch 'unstable' into fix-4929
dknopik Sep 11, 2024
fdb4eca
Merge branch 'unstable' into fix-4929
dknopik Sep 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions beacon_node/beacon_chain/src/attestation_rewards.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use state_processing::per_epoch_processing::altair::{
};
use state_processing::per_epoch_processing::base::rewards_and_penalties::{
get_attestation_component_delta, get_attestation_deltas_all, get_attestation_deltas_subset,
get_inactivity_penalty_delta, get_inclusion_delay_delta,
get_inactivity_penalty_delta, get_inclusion_delay_delta, ProposerRewardCalculation,
};
use state_processing::per_epoch_processing::base::validator_statuses::InclusionInfo;
use state_processing::per_epoch_processing::base::{
Expand Down Expand Up @@ -81,13 +81,24 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
self.compute_ideal_rewards_base(&state, &validator_statuses.total_balances)?;

let indices_to_attestation_delta = if validators.is_empty() {
get_attestation_deltas_all(&state, &validator_statuses, spec)?
.into_iter()
.enumerate()
.collect()
get_attestation_deltas_all(
&state,
&validator_statuses,
ProposerRewardCalculation::Exclude,
spec,
)?
.into_iter()
.enumerate()
.collect()
} else {
let validator_indices = Self::validators_ids_to_indices(&mut state, validators)?;
get_attestation_deltas_subset(&state, &validator_statuses, &validator_indices, spec)?
get_attestation_deltas_subset(
&state,
&validator_statuses,
ProposerRewardCalculation::Exclude,
&validator_indices,
spec,
)?
};

let mut total_rewards = vec![];
Expand Down
129 changes: 110 additions & 19 deletions beacon_node/beacon_chain/src/beacon_block_reward.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,32 @@
use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
use crate::{BeaconChain, BeaconChainError, BeaconChainTypes, StateSkipConfig};
use attesting_indices_base::get_attesting_indices;
use eth2::lighthouse::StandardBlockReward;
use operation_pool::RewardCache;
use safe_arith::SafeArith;
use slog::error;
use state_processing::common::attesting_indices_base;
use state_processing::{
common::{get_attestation_participation_flag_indices, get_attesting_indices_from_state},
common::{
base::{self, SqrtTotalActiveBalance},
get_attestation_participation_flag_indices, get_attesting_indices_from_state,
},
epoch_cache::initialize_epoch_cache,
per_block_processing::{
altair::sync_committee::compute_sync_aggregate_rewards, get_slashable_indices,
},
};
use std::collections::HashSet;
use store::{
consts::altair::{PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, WEIGHT_DENOMINATOR},
RelativeEpoch,
};
use types::{AbstractExecPayload, BeaconBlockRef, BeaconState, BeaconStateError, Hash256};
use types::{AbstractExecPayload, BeaconBlockRef, BeaconState, BeaconStateError, EthSpec};

type BeaconBlockSubRewardValue = u64;

impl<T: BeaconChainTypes> BeaconChain<T> {
pub fn compute_beacon_block_reward<Payload: AbstractExecPayload<T::EthSpec>>(
&self,
block: BeaconBlockRef<'_, T::EthSpec, Payload>,
block_root: Hash256,
state: &mut BeaconState<T::EthSpec>,
) -> Result<StandardBlockReward, BeaconChainError> {
if block.slot() != state.slot() {
Expand All @@ -33,15 +37,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
initialize_epoch_cache(state, &self.spec)?;

self.compute_beacon_block_reward_with_cache(block, block_root, state)
self.compute_beacon_block_reward_with_cache(block, state)
}

// This should only be called after a committee cache has been built
// for both the previous and current epoch
fn compute_beacon_block_reward_with_cache<Payload: AbstractExecPayload<T::EthSpec>>(
&self,
block: BeaconBlockRef<'_, T::EthSpec, Payload>,
block_root: Hash256,
state: &BeaconState<T::EthSpec>,
) -> Result<StandardBlockReward, BeaconChainError> {
let proposer_index = block.proposer_index();
Expand Down Expand Up @@ -72,7 +75,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
})?;

let block_attestation_reward = if let BeaconState::Base(_) = state {
self.compute_beacon_block_attestation_reward_base(block, block_root, state)
self.compute_beacon_block_attestation_reward_base(block, state)
.map_err(|e| {
error!(
self.log,
Expand Down Expand Up @@ -169,19 +172,85 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
fn compute_beacon_block_attestation_reward_base<Payload: AbstractExecPayload<T::EthSpec>>(
&self,
block: BeaconBlockRef<'_, T::EthSpec, Payload>,
block_root: Hash256,
state: &BeaconState<T::EthSpec>,
) -> Result<BeaconBlockSubRewardValue, BeaconChainError> {
// Call compute_block_reward in the base case
// Since base does not have sync aggregate, we only grab attesation portion of the returned
// value
let mut reward_cache = RewardCache::default();
let block_attestation_reward = self
.compute_block_reward(block, block_root, state, &mut reward_cache, true)?
.attestation_rewards
.total;

Ok(block_attestation_reward)
// In phase0, rewards for including attestations are awarded at epoch boundaries when the corresponding
// attestations are contained in state.previous_epoch_attestations. So, if an attestation within this block has
// target = previous_epoch, it is directly inserted into previous_epoch_attestations and we need the state at
// the end of this epoch, or the attestation has target = current_epoch and thus we need the state at the end
// of the next epoch.
// We fetch these lazily, as only one might be needed depending on the block's content.
let mut current_epoch_end = None;
let mut next_epoch_end = None;

let epoch = block.epoch();
let mut block_reward = 0;

let mut rewarded_attesters = HashSet::new();

for attestation in block.body().attestations() {
let processing_epoch_end = if attestation.data().target.epoch == epoch {
let next_epoch_end = match &mut next_epoch_end {
Some(next_epoch_end) => next_epoch_end,
None => {
let state = self.state_at_slot(
epoch.safe_add(1)?.end_slot(T::EthSpec::slots_per_epoch()),
StateSkipConfig::WithoutStateRoots,
)?;
next_epoch_end.get_or_insert(state)
}
};

// If the next epoch end is no longer phase0, no proposer rewards are awarded, as Altair epoch boundry
// processing kicks in. We check this here, as we know that current_epoch_end will always be phase0.
if !matches!(next_epoch_end, BeaconState::Base(_)) {
continue;
}

next_epoch_end
} else if attestation.data().target.epoch == epoch.safe_sub(1)? {
match &mut current_epoch_end {
Some(current_epoch_end) => current_epoch_end,
None => {
let state = self.state_at_slot(
epoch.end_slot(T::EthSpec::slots_per_epoch()),
StateSkipConfig::WithoutStateRoots,
)?;
current_epoch_end.get_or_insert(state)
}
}
} else {
return Err(BeaconChainError::BlockRewardAttestationError);
};

let inclusion_delay = state.slot().safe_sub(attestation.data().slot)?.as_u64();
let sqrt_total_active_balance =
SqrtTotalActiveBalance::new(processing_epoch_end.get_total_active_balance()?);
for attester in get_attesting_indices_from_state(state, attestation)? {
let validator = processing_epoch_end.get_validator(attester as usize)?;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This algorithm is not quite right because it pays out for attestations regardless of whether this proposer was the the first one to include them.

In the spec, the epoch processing function iterates the previous_epoch_attestations and pays the proposer reward to the proposer who includes each attester's attestation with minimal inclusion_delay:

attestation = min([
    a for a in matching_source_attestations
    if index in get_attesting_indices(state, a)
], key=lambda a: a.inclusion_delay)
rewards[attestation.proposer_index] += get_proposer_reward(state, index)

It's quite common that a validator's attestation will be included multiple times in different aggregates, e.g. first at slot 10 and then again at slot 11. Only the proposer of the block of slot 10 gets the reward in this case (the slot 11 proposer either gets nothing for the attestation if it covered no new validators, or just the rewards for the newly covered validators for which this is their first inclusion on chain).

I think to remedy this the simplest fix would be to:

  • For each attester in each attestation included: check whether this is the inclusion with minimal inclusion_delay (usually the first inclusion, but not always). This would involve checking first against state.previous_epoch_attestations/state.current_epoch_attestations as appropriate and then checking against attestations already processed in this block (we don't want to double pay for multi inclusions).

One complication is the min by inclusion_delay. It would be helpful if a property like this held:

If an attestation is included for validator V at slot N in epoch N // 32 then it has a lower inclusion_delay than any other attestation from V included at a slot M > N.

Unfortunately this property does not hold in the case of slashable attestations made by validator V when the chain is not finalising promptly, in which case it could be that:

  • There are 2+ chains with different shufflings that descend from the same source checkpoint (this requires finality to lapse for >2 epochs as shufflings only diverge if their common ancestor block is >2 epochs ago).
  • Validator V signs two slashable attestations in epoch N // 32: one on each chain. Let M = N + 1 for simplicity. On the first chain they attest at e.g. slot N and on the second chain they attest at slot N - 5. The attestation from slot N - 5 is included at N for an inclusion_delay of 5, then the attestation from slot N is included at slot M = N + 1 for an inclusion_delay of 1. So it's possible for an attestation included at a later block to have a lower inclusion_delay, i.e. the property is false.

Therefore in the general case it's not safe to just use the state at the block's slot (state) to infer the rewards. I had hoped that if this property held we could use it to avoid loading the current_epoch_end/next_epoch_end states.

Instead I think we should just proceed with loading all 3 states most of the time and rely on the fact that all the phase0 states are quite small and should load quickly with the soon-to-be-merged hierarchical state diffs:

Copy link
Contributor Author

@dknopik dknopik Sep 11, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the review!
I've implemented a potential fix and a (rather messy) test for this.

I have a question about the second part: While such a pair of attestations would be obviously slashable, would they be really be includable into the "other" chain, respectively? As the shuffling is different, the attesting validator would likely be in a different committee for a different slot in each chain and therefore the validation would fail (for one of the attestations), as the signature does not match the expected validator, as per my understanding.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The beacon node is able to verify attestations from multiple chains, it will load the relevant state & committees and use those to verify the signature. Only in the case where it views the blocks from the other chain as completely invalid will it fail to process the blocks & attestations from that chain.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh no actually you're right. We can't include the attestations from the other fork on chain once the shufflings diverge, because process_attestation in the spec uses the committees from the current chain

I think this means the property I stated does hold. Which means we could use the passed in state for all of the calculations 🤔

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While I currently also think that your statement holds, I don't see why we can avoid the state loads.

In phase0, attestation rewards are processed at the epoch boundary. Therefore, we need the state, as a validator might be slashed at the epoch boundary but not at slot N where the attestation is included. Furthermore, we need the effective balance and total effective balance from the epoch boundary to properly calculate rewards.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oooh yes, the slashing thing definitely prevents us from using the pre-state. Good catch.

The effective balances are immutable within an epoch, but you're right they'd also be a problem for the current epoch attestations we include which don't get rewarded until the end of the next epoch. Let's leave it as-is 😅

if !validator.slashed
&& !rewarded_attesters.contains(&attester)
&& !has_earlier_attestation(
state,
processing_epoch_end,
inclusion_delay,
attester,
)?
{
let base_reward = base::get_base_reward(
validator.effective_balance,
sqrt_total_active_balance,
&self.spec,
)?;
let proposer_reward =
base_reward.safe_div(self.spec.proposer_reward_quotient)?;
block_reward.safe_add_assign(proposer_reward)?;
rewarded_attesters.insert(attester);
}
}
}

Ok(block_reward)
}

fn compute_beacon_block_attestation_reward_altair_deneb<
Expand Down Expand Up @@ -244,3 +313,25 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(total_proposer_reward)
}
}

fn has_earlier_attestation<E: EthSpec>(
state: &BeaconState<E>,
processing_epoch_end: &BeaconState<E>,
inclusion_delay: u64,
attester: u64,
) -> Result<bool, BeaconChainError> {
if inclusion_delay > 1 {
for epoch_att in processing_epoch_end.previous_epoch_attestations()? {
if epoch_att.inclusion_delay < inclusion_delay {
let committee =
state.get_beacon_committee(epoch_att.data.slot, epoch_att.data.index)?;
let earlier_attesters =
get_attesting_indices::<E>(committee.committee, &epoch_att.aggregation_bits)?;
if earlier_attesters.contains(&attester) {
return Ok(true);
}
}
}
}
Ok(false)
}
2 changes: 1 addition & 1 deletion beacon_node/beacon_chain/src/beacon_chain.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5639,7 +5639,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let mut ctxt = ConsensusContext::new(block.slot());

let consensus_block_value = self
.compute_beacon_block_reward(block.message(), Hash256::zero(), &mut state)
.compute_beacon_block_reward(block.message(), &mut state)
.map(|reward| reward.total)
.unwrap_or(0);

Expand Down
Loading
Loading