Historical batches

setup.py

@@ -660,6 +660,7 @@ def wrapper(*args, **kw):  # type: ignore
 process_withdrawals = no_op(process_withdrawals)
 process_bls_to_execution_change = no_op(process_bls_to_execution_change)
 get_expected_withdrawals = get_empty_list_result(get_expected_withdrawals)
+process_historical_batches_update = no_op(process_historical_batches_update)
 
 
 #

specs/capella/beacon-chain.md

@@ -18,6 +18,7 @@
     - [`Withdrawal`](#withdrawal)
     - [`BLSToExecutionChange`](#blstoexecutionchange)
     - [`SignedBLSToExecutionChange`](#signedblstoexecutionchange)
+    - [`HistoricalBatchSummary`](#historicalbatchsummary)
   - [Extended Containers](#extended-containers)
     - [`ExecutionPayload`](#executionpayload)
     - [`ExecutionPayloadHeader`](#executionpayloadheader)
@@ -29,6 +30,8 @@
     - [`is_fully_withdrawable_validator`](#is_fully_withdrawable_validator)
     - [`is_partially_withdrawable_validator`](#is_partially_withdrawable_validator)
 - [Beacon chain state transition function](#beacon-chain-state-transition-function)
+  - [Epoch processing](#epoch-processing)
+    - [Historical batches updates](#historical-batches-updates)
   - [Block processing](#block-processing)
     - [New `get_expected_withdrawals`](#new-get_expected_withdrawals)
     - [New `process_withdrawals`](#new-process_withdrawals)
@@ -115,6 +118,18 @@ class SignedBLSToExecutionChange(Container):
     signature: BLSSignature
 ```
 
+#### `HistoricalBatchSummary`
+
+```python
+class HistoricalBatchSummary(Container):
+    """
+    `HistoricalBatchSummary` matches the components of the phase0 HistoricalBatch
+    making the two hash_tree_root-compatible.
+    """
+    block_batch_root: Root
+    state_batch_root: Root
+```
+
 ### Extended Containers
 
 #### `ExecutionPayload`
@@ -196,7 +211,7 @@ class BeaconState(Container):
     latest_block_header: BeaconBlockHeader
     block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
     state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
-    historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT]
+    historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT]  # Frozen in Merge, replaced by historical_batches
     # Eth1
     eth1_data: Eth1Data
     eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH]
@@ -226,6 +241,8 @@ class BeaconState(Container):
     # Withdrawals
     next_withdrawal_index: WithdrawalIndex  # [New in Capella]
     next_withdrawal_validator_index: ValidatorIndex  # [New in Capella]
+    # Deep history valid from Capella onwards
+    historical_batches: List[HistoricalBatchSummary, HISTORICAL_ROOTS_LIMIT]  # [New in Capella]
 ```
 
 ## Helpers
@@ -270,6 +287,40 @@ def is_partially_withdrawable_validator(validator: Validator, balance: Gwei) ->
 
 ## Beacon chain state transition function
 
+### Epoch processing
+
+*Note*: The function `process_historical_batches_update` replaces `process_historical_roots_update` in Bellatrix.
+
+```python
+def process_epoch(state: BeaconState) -> None:
+    process_justification_and_finalization(state)
+    process_inactivity_updates(state)
+    process_rewards_and_penalties(state)
+    process_registry_updates(state)
+    process_slashings(state)
+    process_eth1_data_reset(state)
+    process_effective_balance_updates(state)
+    process_slashings_reset(state)
+    process_randao_mixes_reset(state)
+    process_historical_batches_update(state)  # [Modified in Capella]
+    process_participation_flag_updates(state)
+    process_sync_committee_updates(state)
+```
+
+#### Historical batches updates
+
+```python
+def process_historical_batches_update(state: BeaconState) -> None:
+    # Set historical block root accumulator.
+    next_epoch = Epoch(get_current_epoch(state) + 1)
+    if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
+        historical_batch = HistoricalBatchSummary(
+            block_batch_root=hash_tree_root(state.block_roots),
+            state_batch_root=hash_tree_root(state.state_roots),
+        )
+        state.historical_batches.append(historical_batch)
+```
+
 ### Block processing
 
 ```python

tests/core/pyspec/eth2spec/test/capella/epoch_processing/test_process_historical_batches_update.py

@@ -0,0 +1,27 @@
+from eth2spec.test.context import (
+    CAPELLA,
+    spec_state_test,
+    with_phases,
+)
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_with
+)
+
+
+def run_process_historical_batches_update(spec, state):
+    yield from run_epoch_processing_with(spec, state, 'process_historical_batches_update')
+
+
+@with_phases([CAPELLA])
+@spec_state_test
+def test_historical_batches_accumulator(spec, state):
+    # skip ahead to near the end of the historical batch period (excl block before epoch processing)
+    state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1
+    pre_historical_batches = state.historical_batches.copy()
+
+    yield from run_process_historical_batches_update(spec, state)
+
+    assert len(state.historical_batches) == len(pre_historical_batches) + 1
+    summary = state.historical_batches[len(state.historical_batches) - 1]
+    assert summary.block_batch_root == state.block_roots.hash_tree_root()
+    assert summary.state_batch_root == state.state_roots.hash_tree_root()

tests/core/pyspec/eth2spec/test/eip4844/epoch_processing/test_process_historical_batches_update.py

@@ -0,0 +1,23 @@
+from eth2spec.test.context import (
+    spec_state_test,
+    with_eip4844_and_later,
+)
+from eth2spec.test.helpers.epoch_processing import (
+    run_epoch_processing_with
+)
+
+
+def run_process_historical_batches_update(spec, state):
+    yield from run_epoch_processing_with(spec, state, 'process_historical_batches_update')
+
+
+@with_eip4844_and_later
+@spec_state_test
+def test_no_op(spec, state):
+    # skip ahead to near the end of the historical batch period (excl block before epoch processing)
+    state.slot = spec.SLOTS_PER_HISTORICAL_ROOT - 1
+    historical_batches_len = len(state.historical_batches)
+
+    yield from run_process_historical_batches_update(spec, state)
+
+    assert len(state.historical_batches) == historical_batches_len

tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py

@@ -1,5 +1,8 @@
 
-from eth2spec.test.helpers.forks import is_post_altair
+from eth2spec.test.helpers.forks import (
+    is_post_altair,
+    is_post_capella,
+)
 
 
 def get_process_calls(spec):
@@ -22,7 +25,10 @@ def get_process_calls(spec):
         'process_effective_balance_updates',
         'process_slashings_reset',
         'process_randao_mixes_reset',
-        'process_historical_roots_update',
+        # Capella replaced `process_historical_roots_update` with `process_historical_batches_update`
+        'process_historical_batches_update' if is_post_capella(spec) else (
+            'process_historical_roots_update'
+        ),
         # Altair replaced `process_participation_record_updates` with `process_participation_flag_updates`
         'process_participation_flag_updates' if is_post_altair(spec) else (
             'process_participation_record_updates'

tests/core/pyspec/eth2spec/test/phase0/epoch_processing/test_process_historical_roots_update.py

@@ -1,4 +1,8 @@
-from eth2spec.test.context import spec_state_test, with_all_phases
+from eth2spec.test.context import (
+    PHASE0, ALTAIR, BELLATRIX,
+    spec_state_test,
+    with_phases,
+)
 from eth2spec.test.helpers.epoch_processing import (
     run_epoch_processing_with
 )
@@ -8,7 +12,7 @@ def run_process_historical_roots_update(spec, state):
     yield from run_epoch_processing_with(spec, state, 'process_historical_roots_update')
 
 
-@with_all_phases
+@with_phases([PHASE0, ALTAIR, BELLATRIX])
 @spec_state_test
 def test_historical_root_accumulator(spec, state):
     # skip ahead to near the end of the historical roots period (excl block before epoch processing)

tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py

@@ -29,8 +29,8 @@
     compute_committee_indices,
     compute_sync_committee_participant_reward_and_penalty,
 )
-from eth2spec.test.helpers.constants import PHASE0, MINIMAL
-from eth2spec.test.helpers.forks import is_post_altair, is_post_bellatrix
+from eth2spec.test.helpers.constants import PHASE0, EIP4844, MINIMAL
+from eth2spec.test.helpers.forks import is_post_altair, is_post_bellatrix, is_post_capella
 from eth2spec.test.context import (
     spec_test, spec_state_test, dump_skipping_message,
     with_phases, with_all_phases, single_phase,
@@ -1026,7 +1026,10 @@ def test_balance_driven_status_transitions(spec, state):
 @always_bls
 def test_historical_batch(spec, state):
     state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1
-    pre_historical_roots_len = len(state.historical_roots)
+    pre_historical_roots = state.historical_roots.copy()
+
+    if is_post_capella(spec):
+        pre_historical_batches = state.historical_batches.copy()
 
     yield 'pre', state
 
@@ -1038,7 +1041,18 @@ def test_historical_batch(spec, state):
 
     assert state.slot == block.slot
     assert spec.get_current_epoch(state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0
-    assert len(state.historical_roots) == pre_historical_roots_len + 1
+
+    # check history update
+    if is_post_capella(spec):
+        # Frozen `historical_roots`
+        assert state.historical_roots == pre_historical_roots
+        if spec.fork == EIP4844:
+            # TODO: no-op for now in EIP4844 testnet
+            assert state.historical_batches == pre_historical_batches
+        else:
+            assert len(state.historical_batches) == len(pre_historical_batches) + 1
+    else:
+        assert len(state.historical_roots) == len(pre_historical_roots) + 1
 
 
 @with_all_phases

tests/core/pyspec/eth2spec/test/phase0/sanity/test_slots.py

@@ -1,3 +1,9 @@
+from eth2spec.test.helpers.constants import (
+    EIP4844,
+)
+from eth2spec.test.helpers.forks import (
+    is_post_capella,
+)
 from eth2spec.test.helpers.state import get_state_root
 from eth2spec.test.context import (
     spec_state_test,
@@ -61,3 +67,30 @@ def test_over_epoch_boundary(spec, state):
     yield 'slots', int(slots)
     spec.process_slots(state, state.slot + slots)
     yield 'post', state
+
+
+@with_all_phases
+@spec_state_test
+def test_historical_accumulator(spec, state):
+    pre_historical_roots = state.historical_roots.copy()
+
+    if is_post_capella(spec):
+        pre_historical_batches = state.historical_batches.copy()
+
+    yield 'pre', state
+    slots = spec.SLOTS_PER_HISTORICAL_ROOT
+    yield 'slots', int(slots)
+    spec.process_slots(state, state.slot + slots)
+    yield 'post', state
+
+    # check history update
+    if is_post_capella(spec):
+        # Frozen `historical_roots`
+        assert state.historical_roots == pre_historical_roots
+        if spec.fork == EIP4844:
+            # TODO: no-op for now in EIP4844 testnet
+            assert state.historical_batches == pre_historical_batches
+        else:
+            assert len(state.historical_batches) == len(pre_historical_batches) + 1
+    else:
+        assert len(state.historical_roots) == len(pre_historical_roots) + 1

tests/generators/epoch_processing/main.py

@@ -27,13 +27,15 @@
     # so no additional tests required.
     bellatrix_mods = altair_mods
 
-    # No epoch-processing changes in Capella and previous testing repeats with new types,
-    # so no additional tests required.
-    capella_mods = bellatrix_mods
+    _new_capella_mods = {key: 'eth2spec.test.capella.epoch_processing.test_process_' + key for key in [
+        'historical_batches_update',
+    ]}
+    capella_mods = combine_mods(_new_capella_mods, bellatrix_mods)
 
-    # No epoch-processing changes in EIP4844 and previous testing repeats with new types,
-    # so no additional tests required.
-    eip4844_mods = capella_mods
+    _new_eip4844_mods = {key: 'eth2spec.test.eip4844.epoch_processing.test_process_' + key for key in [
+        'historical_batches_update',
+    ]}
+    eip4844_mods = combine_mods(_new_eip4844_mods, capella_mods)
 
     # TODO Custody Game testgen is disabled for now
     # custody_game_mods = {**{key: 'eth2spec.test.custody_game.epoch_processing.test_process_' + key for key in [