Allow honest validators to reorg late blocks

pysetup/spec_builders/bellatrix.py

@@ -27,7 +27,12 @@ def get_execution_state(_execution_state_root: Bytes32) -> ExecutionState:
 
 
 def get_pow_chain_head() -> PowBlock:
-    pass"""
+    pass
+
+
+def validator_is_connected(validator_index: ValidatorIndex) -> bool:
+    # pylint: disable=unused-argument
+    return True"""
 
     @classmethod
     def execution_engine_cls(cls) -> str:

specs/bellatrix/fork-choice.md

@@ -11,6 +11,7 @@
   - [`ExecutionEngine`](#executionengine)
     - [`notify_forkchoice_updated`](#notify_forkchoice_updated)
       - [`safe_block_hash`](#safe_block_hash)
+      - [`should_override_forkchoice_update`](#should_override_forkchoice_update)
 - [Helpers](#helpers)
   - [`PayloadAttributes`](#payloadattributes)
   - [`PowBlock`](#powblock)
@@ -76,6 +77,85 @@ As per EIP-3675, before a post-transition block is finalized, `notify_forkchoice
 The `safe_block_hash` parameter MUST be set to return value of
 [`get_safe_execution_payload_hash(store: Store)`](../../fork_choice/safe-block.md#get_safe_execution_payload_hash) function.
 
+##### `should_override_forkchoice_update`
+
+If proposer boost re-orgs are implemented and enabled (see `get_proposer_head`) then additional care
+must be taken to ensure that the proposer is able to build an execution payload.
+
+If a beacon node knows it will propose the next block then it SHOULD NOT call
+`notify_forkchoice_updated` if it detects the current head to be weak and potentially capable of
+being re-orged. Complete information for evaluating `get_proposer_head` _will not_ be available
+immediately after the receipt of a new block, so an approximation of those conditions should be
+used when deciding whether to send or suppress a fork choice notification. The exact conditions
+used may be implementation-specific, a suggested implementation is below.
+
+Let `validator_is_connected(validator_index: ValidatorIndex) -> bool` be a function that indicates
+whether the validator with `validator_index` is connected to the node (e.g. has sent an unexpired
+proposer preparation message).
+
+```python
+def should_override_forkchoice_update(store: Store, head_root: Root) -> bool:
+    head_block = store.blocks[head_root]
+    parent_root = head_block.parent_root
+    parent_block = store.blocks[parent_root]
+    current_slot = get_current_slot(store)
+    proposal_slot = head_block.slot + Slot(1)
+
+    # Only re-org the head_block block if it arrived later than the attestation deadline.
+    head_late = is_head_late(store, head_root)
+
+    # Shuffling stable.
+    shuffling_stable = is_shuffling_stable(proposal_slot)
+
+    # FFG information of the new head_block will be competitive with the current head.
+    ffg_competitive = is_ffg_competitive(store, head_root, parent_root)
+
+    # Only suppress the fork choice update if we are confident that we will propose the next block.
+    parent_state_advanced = store.block_states[parent_root]
+    process_slots(parent_state_advanced, proposal_slot)
+    proposer_index = get_beacon_proposer_index(parent_state_advanced)
+    proposing_reorg_slot = validator_is_connected(proposer_index)
+
+    # Do not re-org if the chain is not finalizing with acceptable frequency.
+    finalization_ok = is_finalization_ok(store, proposal_slot)
+
+    # Single slot re-org.
+    parent_slot_ok = parent_block.slot + 1 == head_block.slot
+    proposing_on_time = is_proposing_on_time(store)
+
+    # Note that this condition is different from `get_proposer_head`
+    current_time_ok = (head_block.slot == current_slot
+                       or (proposal_slot == current_slot and proposing_on_time))
+    single_slot_reorg = parent_slot_ok and current_time_ok
+
+    # Check the head weight only if the attestations from the head slot have already been applied.
+    # Implementations may want to do this in different ways, e.g. by advancing
+    # `store.time` early, or by counting queued attestations during the head block's slot.
+    if current_slot > head_block.slot:
+        head_weak = is_head_weak(store, head_root)
+        parent_strong = is_parent_strong(store, parent_root)
+    else:
+        head_weak = True
+        parent_strong = True
+
+    return all([head_late, shuffling_stable, proposing_reorg_slot, ffg_competitive, finalization_ok,
+                single_slot_reorg, head_weak, parent_strong])
+```
+
+*Note*: The ordering of conditions is a suggestion only. Implementations are free to
+optimize by re-ordering the conditions from least to most expensive and by returning early if
+any of the early conditions are `False`.
+
+In case `should_override_forkchoice_update` returns `True`, a node SHOULD instead call
+`notify_forkchoice_updated` with parameters appropriate for building upon the parent block. Care
+must be taken to compute the correct `payload_attributes`, as they may change depending on the slot
+of the block to be proposed (due to withdrawals).
+
+If `should_override_forkchoice_update` returns `True` but `get_proposer_head` later chooses the
+canonical head rather than its parent, then this is a misprediction that will cause the node
+to construct a payload with less notice. The result of `get_proposer_head` MUST be honoured in
+preference to the heuristic method.
+
 ## Helpers
 
 ### `PayloadAttributes`
@@ -191,11 +271,15 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     # Add new state for this block to the store
     store.block_states[block_root] = state
 
-    # Add proposer score boost if the block is timely
+    # Add block timeliness to the store
     time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT
     is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT
+    is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval
+    store.block_timeliness[hash_tree_root(block)] = is_timely
+
+    # Add proposer score boost if the block is timely and not conflicting with an existing block
     is_first_block = store.proposer_boost_root == Root()
-    if get_current_slot(store) == block.slot and is_before_attesting_interval and is_first_block:
+    if is_timely and is_first_block:
         store.proposer_boost_root = hash_tree_root(block)
 
     # Update checkpoints in store if necessary

specs/capella/fork-choice.md

@@ -103,11 +103,15 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     # Add new state for this block to the store
     store.block_states[block_root] = state
 
-    # Add proposer score boost if the block is timely
+    # Add block timeliness to the store
     time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT
     is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT
+    is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval
+    store.block_timeliness[hash_tree_root(block)] = is_timely
+
+    # Add proposer score boost if the block is timely and not conflicting with an existing block
     is_first_block = store.proposer_boost_root == Root()
-    if get_current_slot(store) == block.slot and is_before_attesting_interval and is_first_block:
+    if is_timely and is_first_block:
         store.proposer_boost_root = hash_tree_root(block)
 
     # Update checkpoints in store if necessary

specs/deneb/fork-choice.md

@@ -103,11 +103,15 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     # Add new state for this block to the store
     store.block_states[block_root] = state
 
-    # Add proposer score boost if the block is timely
+    # Add block timeliness to the store
     time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT
     is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT
+    is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval
+    store.block_timeliness[hash_tree_root(block)] = is_timely
+
+    # Add proposer score boost if the block is timely and not conflicting with an existing block
     is_first_block = store.proposer_boost_root == Root()
-    if get_current_slot(store) == block.slot and is_before_attesting_interval and is_first_block:
+    if is_timely and is_first_block:
         store.proposer_boost_root = hash_tree_root(block)
 
     # Update checkpoints in store if necessary

specs/phase0/fork-choice.md

@@ -18,6 +18,7 @@
     - [`get_current_slot`](#get_current_slot)
     - [`compute_slots_since_epoch_start`](#compute_slots_since_epoch_start)
     - [`get_ancestor`](#get_ancestor)
+    - [`calculate_committee_fraction`](#calculate_committee_fraction)
     - [`get_checkpoint_block`](#get_checkpoint_block)
     - [`get_weight`](#get_weight)
     - [`get_voting_source`](#get_voting_source)
@@ -26,6 +27,15 @@
     - [`get_head`](#get_head)
     - [`update_checkpoints`](#update_checkpoints)
     - [`update_unrealized_checkpoints`](#update_unrealized_checkpoints)
+    - [Proposer head and reorg helpers](#proposer-head-and-reorg-helpers)
+      - [`is_head_late`](#is_head_late)
+      - [`is_shuffling_stable`](#is_shuffling_stable)
+      - [`is_ffg_competitive`](#is_ffg_competitive)
+      - [`is_finalization_ok`](#is_finalization_ok)
+      - [`is_proposing_on_time`](#is_proposing_on_time)
+      - [`is_head_weak`](#is_head_weak)
+      - [`is_parent_strong`](#is_parent_strong)
+      - [`get_proposer_head`](#get_proposer_head)
     - [Pull-up tip helpers](#pull-up-tip-helpers)
       - [`compute_pulled_up_tip`](#compute_pulled_up_tip)
     - [`on_tick` helpers](#on_tick-helpers)
@@ -76,11 +86,16 @@ Any of the above handlers that trigger an unhandled exception (e.g. a failed ass
 
 ### Configuration
 
-| Name                   | Value        |
-| ---------------------- | ------------ |
-| `PROPOSER_SCORE_BOOST` | `uint64(40)` |
+| Name                                  | Value        |
+| ------------------------------------- | ------------ |
+| `PROPOSER_SCORE_BOOST`                | `uint64(40)` |
+| `REORG_WEIGHT_THRESHOLD`              | `uint64(20)` |
+| `REORG_PARENT_WEIGHT_THRESHOLD`       | `uint64(160)`|
+| `REORG_MAX_EPOCHS_SINCE_FINALIZATION` | `Epoch(2)`   |
 
-- The proposer score boost is worth `PROPOSER_SCORE_BOOST` percentage of the committee's weight, i.e., for slot with committee weight `committee_weight` the boost weight is equal to `(committee_weight * PROPOSER_SCORE_BOOST) // 100`.
+- The proposer score boost and re-org weight threshold are percentage
+  values that are measured with respect to the weight of a single committee. See
+  `calculate_committee_fraction`.
 
 ### Helpers
 
@@ -115,6 +130,7 @@ class Store(object):
     equivocating_indices: Set[ValidatorIndex]
     blocks: Dict[Root, BeaconBlock] = field(default_factory=dict)
     block_states: Dict[Root, BeaconState] = field(default_factory=dict)
+    block_timeliness: Dict[Root, boolean] = field(default_factory=dict)
     checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict)
     latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict)
     unrealized_justifications: Dict[Root, Checkpoint] = field(default_factory=dict)
@@ -191,6 +207,14 @@ def get_ancestor(store: Store, root: Root, slot: Slot) -> Root:
     return root
 ```
 
+#### `calculate_committee_fraction`
+
+```python
+def calculate_committee_fraction(state: BeaconState, committee_percent: uint64) -> Gwei:
+    committee_weight = get_total_active_balance(state) // SLOTS_PER_EPOCH
+    return Gwei((committee_weight * committee_percent) // 100)
+```
+
 #### `get_checkpoint_block`
 
 ```python
@@ -225,8 +249,7 @@ def get_weight(store: Store, root: Root) -> Gwei:
     proposer_score = Gwei(0)
     # Boost is applied if ``root`` is an ancestor of ``proposer_boost_root``
     if get_ancestor(store, store.proposer_boost_root, store.blocks[root].slot) == root:
-        committee_weight = get_total_active_balance(state) // SLOTS_PER_EPOCH
-        proposer_score = (committee_weight * PROPOSER_SCORE_BOOST) // 100
+        proposer_score = calculate_committee_fraction(state, PROPOSER_SCORE_BOOST)
     return attestation_score + proposer_score
 ```
 
@@ -247,7 +270,6 @@ def get_voting_source(store: Store, block_root: Root) -> Checkpoint:
         # The block is not from a prior epoch, therefore the voting source is not pulled up
         head_state = store.block_states[block_root]
         return head_state.current_justified_checkpoint
-
 ```
 
 #### `filter_block_tree`
@@ -374,7 +396,112 @@ def update_unrealized_checkpoints(store: Store, unrealized_justified_checkpoint:
     if unrealized_finalized_checkpoint.epoch > store.unrealized_finalized_checkpoint.epoch:
         store.unrealized_finalized_checkpoint = unrealized_finalized_checkpoint
 ```
+#### Proposer head and reorg helpers
+
+_Implementing these helpers is optional_.
+
+##### `is_head_late`
+```python
+def is_head_late(store: Store, head_root: Root) -> bool:
+    return not store.block_timeliness[head_root]
+```
+
+##### `is_shuffling_stable`
+```python
+def is_shuffling_stable(slot: Slot) -> bool:
+    return slot % SLOTS_PER_EPOCH != 0
+```
+
+##### `is_ffg_competitive`
+
+```python
+def is_ffg_competitive(store: Store, head_root: Root, parent_root: Root) -> bool:
+    return (store.unrealized_justifications[head_root] == store.unrealized_justifications[parent_root])
+```
+
+##### `is_finalization_ok`
+
+```python
+def is_finalization_ok(store: Store, slot: Slot) -> bool:
+    epochs_since_finalization = compute_epoch_at_slot(slot) - store.finalized_checkpoint.epoch
+    return epochs_since_finalization <= REORG_MAX_EPOCHS_SINCE_FINALIZATION
+```
+
+##### `is_proposing_on_time`
+
+```python
+def is_proposing_on_time(store: Store) -> bool:
+    time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT
+    proposer_reorg_cutoff = SECONDS_PER_SLOT // INTERVALS_PER_SLOT // 2
+    return time_into_slot <= proposer_reorg_cutoff
+```
+
+##### `is_head_weak`
+
+```python
+def is_head_weak(store: Store, head_root: Root) -> bool:
+    justified_state = store.checkpoint_states[store.justified_checkpoint]
+    reorg_threshold = calculate_committee_fraction(justified_state, REORG_WEIGHT_THRESHOLD)
+    head_weight = get_weight(store, head_root)
+    return head_weight < reorg_threshold
+```
+
+##### `is_parent_strong`
+
+```python
+def is_parent_strong(store: Store, parent_root: Root) -> bool:
+    justified_state = store.checkpoint_states[store.justified_checkpoint]
+    parent_threshold = calculate_committee_fraction(justified_state, REORG_PARENT_WEIGHT_THRESHOLD)
+    parent_weight = get_weight(store, parent_root)
+    return parent_weight > parent_threshold
+```
+
+##### `get_proposer_head`
+
+```python
+def get_proposer_head(store: Store, head_root: Root, slot: Slot) -> Root:
+    head_block = store.blocks[head_root]
+    parent_root = head_block.parent_root
+    parent_block = store.blocks[parent_root]
+
+    # Only re-org the head block if it arrived later than the attestation deadline.
+    head_late = is_head_late(store, head_root)
+
+    # Do not re-org on an epoch boundary where the proposer shuffling could change.
+    shuffling_stable = is_shuffling_stable(slot)
+
+    # Ensure that the FFG information of the new head will be competitive with the current head.
+    ffg_competitive = is_ffg_competitive(store, head_root, parent_root)
+
+    # Do not re-org if the chain is not finalizing with acceptable frequency.
+    finalization_ok = is_finalization_ok(store, slot)
+
+    # Only re-org if we are proposing on-time.
+    proposing_on_time = is_proposing_on_time(store)
+
+    # Only re-org a single slot at most.
+    parent_slot_ok = parent_block.slot + 1 == head_block.slot
+    current_time_ok = head_block.slot + 1 == slot
+    single_slot_reorg = parent_slot_ok and current_time_ok
+
+    # Check that the head has few enough votes to be overpowered by our proposer boost.
+    assert store.proposer_boost_root != head_root  # ensure boost has worn off
+    head_weak = is_head_weak(store, head_root)
+
+    # Check that the missing votes are assigned to the parent and not being hoarded.
+    parent_strong = is_parent_strong(store, parent_root)
+
+    if all([head_late, shuffling_stable, ffg_competitive, finalization_ok,
+            proposing_on_time, single_slot_reorg, head_weak, parent_strong]):
+        # We can re-org the current head by building upon its parent block.
+        return parent_root
+    else:
+        return head_root
+```
 
+*Note*: The ordering of conditions is a suggestion only. Implementations are free to
+optimize by re-ordering the conditions from least to most expensive and by returning early if
+any of the early conditions are `False`.
 
 #### Pull-up tip helpers
 
@@ -536,11 +663,15 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     # Add new state for this block to the store
     store.block_states[block_root] = state
 
-    # Add proposer score boost if the block is timely
+    # Add block timeliness to the store
     time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT
     is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT
+    is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval
+    store.block_timeliness[hash_tree_root(block)] = is_timely
+
+    # Add proposer score boost if the block is timely and not conflicting with an existing block
     is_first_block = store.proposer_boost_root == Root()
-    if get_current_slot(store) == block.slot and is_before_attesting_interval and is_first_block:
+    if is_timely and is_first_block:
         store.proposer_boost_root = hash_tree_root(block)
 
     # Update checkpoints in store if necessary