Spec update: remove BeaconState.shard_committees_at_slots in favor of crosslink committees; validator_registry_latest_change_slot -> validator_registry_update_slot mechanical renaming

beacon_chain/beacon_node.nim

@@ -1,7 +1,7 @@
 import
   std_shims/[os_shims, objects], net, sequtils, options, tables,
   asyncdispatch2, chronicles, confutils, eth_p2p, eth_keys,
-  spec/[datatypes, digest, crypto, beaconstate, helpers], conf, time,
+  spec/[datatypes, digest, crypto, beaconstate, helpers, validator], conf, time,
   state_transition, fork_choice, ssz, beacon_chain_db, validator_pool, extras,
   mainchain_monitor, sync_protocol, gossipsub_protocol, trusted_state_snapshots,
   eth_trie/db, eth_trie/backends/rocksdb_backend
@@ -301,11 +301,14 @@ proc scheduleEpochActions(node: BeaconNode, epoch: uint64) =
     let
       committeesIdx = get_shard_committees_index(nextState, slot)
 
-    for shard in node.beaconState.shard_committees_at_slots[committees_idx]:
-      for i, validatorIdx in shard.committee:
+    #for shard in node.beaconState.shard_committees_at_slots[committees_idx]:
+    for crosslink_committee in get_crosslink_committees_at_slot(node.beaconState, committees_idx):
+      #for i, validatorIdx in shard.committee:
+      for i, validatorIdx in crosslink_committee.a:
         let validator = node.getAttachedValidator(validatorIdx)
         if validator != nil:
-          scheduleAttestation(node, validator, slot, shard.shard, shard.committee.len, i)
+          #scheduleAttestation(node, validator, slot, shard.shard, shard.committee.len, i)
+          scheduleAttestation(node, validator, slot, crosslink_committee.b, crosslink_committee.a.len, i)
 
   node.lastScheduledEpoch = epoch
   let

beacon_chain/spec/beaconstate.nim

@@ -217,7 +217,7 @@ func get_initial_beacon_state*(
         fork_slot: GENESIS_SLOT,
     ),
 
-    validator_registry_latest_change_slot: GENESIS_SLOT,
+    validator_registry_update_slot: GENESIS_SLOT,
     validator_registry_exit_count: 0,
     validator_registry_delta_chain_tip: ZERO_HASH,
 
@@ -260,30 +260,6 @@ func get_initial_beacon_state*(
     if get_effective_balance(state, vi) > MAX_DEPOSIT_AMOUNT:
       activate_validator(state, vi, true)
 
-  # initial_shuffling + initial_shuffling in spec, but more ugly
-  # TODO remove temporary workaround
-  # previously, shuffling created foo[slot][committee_per_slot]
-  # now that's flattened to [committee_0_slot_0, c_1_s_0, ..., c_2_s_1, c_3_s_1, ...]
-  # so build adapter to keep this working until full conversion to current spec
-  # target structure is array[2 * EPOCH_LENGTH, seq[ShardCommittee]],
-  # where ShardCommittee is: shard*: uint64 / committee*: seq[Uint24]
-  let
-    initial_shuffling =
-      get_shuffling(Eth2Digest(), state.validator_registry, state.slot)
-    committee_count_per_slot = initial_shuffling.len div EPOCH_LENGTH
-
-  for i in 0 ..< EPOCH_LENGTH:
-    state.shard_committees_at_slots[i] = @[]
-    state.shard_committees_at_slots[EPOCH_LENGTH + i] = @[]
-
-  for i, committee2 in initial_shuffling:
-    let slot = i div committee_count_per_slot
-    var sc: ShardCommittee
-    sc.shard = i.uint64
-    sc.committee = committee2
-    state.shard_committees_at_slots[slot] = concat(state.shard_committees_at_slots[slot], @[sc])
-    state.shard_committees_at_slots[EPOCH_LENGTH + slot] = concat(state.shard_committees_at_slots[EPOCH_LENGTH + slot], @[sc])
-
   state
 
 func get_block_root*(state: BeaconState,
@@ -301,28 +277,33 @@ func get_randao_mix*(state: BeaconState,
 
 func get_attestation_participants*(state: BeaconState,
                                    attestation_data: AttestationData,
-                                   participation_bitfield: seq[byte]): seq[Uint24] =
+                                   aggregation_bitfield: seq[byte]): seq[Uint24] =
   ## Attestation participants in the attestation data are called out in a
   ## bit field that corresponds to the committee of the shard at the time - this
   ## function converts it to list of indices in to BeaconState.validators
   ## Returns empty list if the shard is not found
   # TODO Linear search through shard list? borderline ok, it's a small list
   # TODO bitfield type needed, once bit order settles down
   # TODO iterator candidate
-  let
-    sncs_for_slot = get_shard_committees_at_slot(
-      state, attestation_data.slot)
-
-  for snc in sncs_for_slot:
-    if snc.shard != attestation_data.shard:
-      continue
-
-    # TODO investigate functional library / approach to help avoid loop bugs
-    assert len(participation_bitfield) == ceil_div8(len(snc.committee))
-    for i, vindex in snc.committee:
-      if bitIsSet(participation_bitfield, i):
-        result.add(vindex)
-    return # found the shard, we're done
+
+  # Find the committee in the list with the desired shard
+  let crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot)
+
+  # TODO investigate functional library / approach to help avoid loop bugs
+  assert any(
+    crosslink_committees,
+    func (x: tuple[a: seq[Uint24], b: uint64]): bool = x[1] == attestation_data.shard)
+  let crosslink_committee = mapIt(
+    filterIt(crosslink_committees, it.b == attestation_data.shard),
+    it.a)[0]
+  assert len(aggregation_bitfield) == (len(crosslink_committee) + 7) div 8
+
+  # Find the participating attesters in the committee
+  result = @[]
+  for i, validator_index in crosslink_committee:
+    let aggregation_bit = (aggregation_bitfield[i div 8] shr (7 - (i mod 8))) mod 2
+    if aggregation_bit == 1:
+      result.add(validator_index)
 
 func process_ejections*(state: var BeaconState) =
   ## Iterate through the validator registry

beacon_chain/spec/datatypes.nim

@@ -307,7 +307,7 @@ type
     validator_balances*: seq[uint64] ##\
     ## Validator balances in Gwei!
 
-    validator_registry_latest_change_slot*: uint64
+    validator_registry_update_slot*: uint64
     validator_registry_exit_count*: uint64
     validator_registry_delta_chain_tip*: Eth2Digest ##\
     ## For light clients to easily track delta
@@ -317,10 +317,6 @@ type
     latest_vdf_outputs*: array[
       (LATEST_RANDAO_MIXES_LENGTH div EPOCH_LENGTH).int, Eth2Digest]
 
-    shard_committees_at_slots*: array[2 * EPOCH_LENGTH, seq[ShardCommittee]] ## \
-    ## Committee members and their assigned shard, per slot, covers 2 cycles
-    ## worth of assignments
-
     previous_epoch_start_shard*: uint64
     current_epoch_start_shard*: uint64
     previous_epoch_calculation_slot*: uint64

beacon_chain/spec/helpers.nim

@@ -101,35 +101,14 @@ func repeat_hash*(v: Eth2Digest, n: SomeInteger): Eth2Digest =
     dec n
 
 func get_shard_committees_index*(state: BeaconState, slot: uint64): uint64 =
+  # TODO temporary adapter; remove when all users gone
   ## Warning: as it stands, this helper only works during state updates _after_
   ## state.slot has been incremented but before shard_committees_at_slots has
   ## been updated!
   # TODO spec unsigned-unsafe here
   doAssert slot + (state.slot mod EPOCH_LENGTH) + EPOCH_LENGTH > state.slot
   slot + (state.slot mod EPOCH_LENGTH) + EPOCH_LENGTH - state.slot
 
-proc get_shard_committees_at_slot*(
-    state: BeaconState, slot: uint64): seq[ShardCommittee] =
-  let index = state.get_shard_committees_index(slot)
-  state.shard_committees_at_slots[index]
-
-func get_beacon_proposer_index*(state: BeaconState, slot: uint64): Uint24 =
-  ## From Casper RPJ mini-spec:
-  ## When slot i begins, validator Vidx is expected
-  ## to create ("propose") a block, which contains a pointer to some parent block
-  ## that they perceive as the "head of the chain",
-  ## and includes all of the **attestations** that they know about
-  ## that have not yet been included into that chain.
-  ##
-  ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee)
-  # TODO this index is invalid outside of the block state transition function
-  #      because presently, `state.slot += 1` happens before this function
-  #      is called - see also testutil.getNextBeaconProposerIndex
-  let idx = get_shard_committees_index(state, slot)
-  doAssert idx.int < state.shard_committees_at_slots.len
-  doAssert state.shard_committees_at_slots[idx].len > 0
-  state.shard_committees_at_slots[idx][0].committee.mod_get(slot)
-
 func integer_squareroot*(n: SomeInteger): SomeInteger =
   ## The largest integer ``x`` such that ``x**2`` is less than ``n``.
   var

beacon_chain/spec/validator.nim

@@ -125,3 +125,31 @@ func get_crosslink_committees_at_slot*(state: BeaconState, slot: uint64) : seq[t
        shuffling[(committees_per_slot * offset + i.uint64).int],
        (slot_start_shard + i.uint64) mod SHARD_COUNT
       )
+
+func get_shard_committees_at_slot*(
+    state: BeaconState, slot: uint64): seq[ShardCommittee] =
+  # TODO temporary adapter; remove when all users gone
+  # where ShardCommittee is: shard*: uint64 / committee*: seq[Uint24]
+  let index = state.get_shard_committees_index(slot)
+  #state.shard_committees_at_slots[index]
+  for crosslink_committee in get_crosslink_committees_at_slot(state, slot):
+    var sac: ShardCommittee
+    sac.shard = crosslink_committee.b
+    sac.committee = crosslink_committee.a
+    result.add sac
+
+func get_beacon_proposer_index*(state: BeaconState, slot: uint64): Uint24 =
+  ## From Casper RPJ mini-spec:
+  ## When slot i begins, validator Vidx is expected
+  ## to create ("propose") a block, which contains a pointer to some parent block
+  ## that they perceive as the "head of the chain",
+  ## and includes all of the **attestations** that they know about
+  ## that have not yet been included into that chain.
+  ##
+  ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee)
+  ## Returns the beacon proposer index for the ``slot``.
+  # TODO this index is invalid outside of the block state transition function
+  #      because presently, `state.slot += 1` happens before this function
+  #      is called - see also testutil.getNextBeaconProposerIndex
+  let first_committee = get_crosslink_committees_at_slot(state, slot)[0][0]
+  first_committee[slot.int mod len(first_committee)]

beacon_chain/state_transition.nim

@@ -538,21 +538,21 @@ func processEpoch(state: var BeaconState) =
   #      these closures outside this scope, but still..
   let statePtr = state.addr
   func attesting_validator_indices(
-      shard_committee: ShardCommittee, shard_block_root: Eth2Digest): seq[Uint24] =
+      crosslink_committee: tuple[a: seq[Uint24], b: uint64], shard_block_root: Eth2Digest): seq[Uint24] =
     let shard_block_attestations =
       concat(current_epoch_attestations, previous_epoch_attestations).
-      filterIt(it.data.shard == shard_committee.shard and
+      filterIt(it.data.shard == crosslink_committee.b and
         it.data.shard_block_root == shard_block_root)
     get_attester_indices(statePtr[], shard_block_attestations)
 
-  func winning_root(shard_committee: ShardCommittee): Eth2Digest =
-    # * Let `winning_root(shard_committee)` be equal to the value of
+  func winning_root(crosslink_committee: tuple[a: seq[Uint24], b: uint64]): Eth2Digest =
+    # * Let `winning_root(crosslink_committee)` be equal to the value of
     #   `shard_block_root` such that
-    #   `sum([get_effective_balance(state, i) for i in attesting_validator_indices(shard_committee, shard_block_root)])`
+    #   `sum([get_effective_balance(state, i) for i in attesting_validator_indices(crosslink_committee, shard_block_root)])`
     #   is maximized (ties broken by favoring lower `shard_block_root` values).
     let candidates =
       concat(current_epoch_attestations, previous_epoch_attestations).
-        filterIt(it.data.shard == shard_committee.shard).
+        filterIt(it.data.shard == crosslink_committee.b).
         mapIt(it.data.shard_block_root)
 
     # TODO not covered by spec!
@@ -562,24 +562,27 @@ func processEpoch(state: var BeaconState) =
     var max_hash = candidates[0]
     var max_val =
       sum_effective_balances(
-        statePtr[], attesting_validator_indices(shard_committee, max_hash))
+        statePtr[], attesting_validator_indices(crosslink_committee, max_hash))
     for candidate in candidates[1..^1]:
       let val = sum_effective_balances(
-        statePtr[], attesting_validator_indices(shard_committee, candidate))
+        statePtr[], attesting_validator_indices(crosslink_committee, candidate))
       if val > max_val or (val == max_val and candidate.lowerThan(max_hash)):
         max_hash = candidate
         max_val = val
     max_hash
 
-  func attesting_validator_indices(shard_committee: ShardCommittee): seq[Uint24] =
-    attesting_validator_indices(shard_committee, winning_root(shard_committee))
+  func attesting_validators(crosslink_committee: tuple[a: seq[Uint24], b: uint64]): seq[Uint24] =
+    attesting_validator_indices(crosslink_committee, winning_root(crosslink_committee))
 
-  func total_attesting_balance(shard_committee: ShardCommittee): uint64 =
+  func attesting_validator_indices(crosslink_committee: tuple[a: seq[Uint24], b: uint64]): seq[Uint24] =
+    attesting_validator_indices(crosslink_committee, winning_root(crosslink_committee))
+
+  func total_attesting_balance(crosslink_committee: tuple[a: seq[Uint24], b: uint64]): uint64 =
     sum_effective_balances(
-      statePtr[], attesting_validator_indices(shard_committee))
+      statePtr[], attesting_validator_indices(crosslink_committee))
 
-  func total_balance_sac(shard_committee: ShardCommittee): uint64 =
-    sum_effective_balances(statePtr[], shard_committee.committee)
+  func total_balance_sac(crosslink_committee: tuple[a: seq[Uint24], b: uint64]): uint64 =
+    sum_effective_balances(statePtr[], crosslink_committee.a)
 
   block: # Eth1 data
     if state.slot mod ETH1_DATA_VOTING_PERIOD == 0:
@@ -689,43 +692,42 @@ func processEpoch(state: var BeaconState) =
         base_reward(state, v) div INCLUDER_REWARD_QUOTIENT
 
   block: # Crosslinks
-    for sac in state.shard_committees_at_slots[0 ..< EPOCH_LENGTH]:
-      for shard_committee in sac:
-        for index in shard_committee.committee:
-          if index in attesting_validator_indices(shard_committee):
-            state.validator_balances[index] +=
-              base_reward(state, index) *
-              total_attesting_balance(shard_committee) div
-              total_balance_sac(shard_committee)
+    for slot in state.slot - 2 * EPOCH_LENGTH ..< state.slot - EPOCH_LENGTH:
+      let crosslink_committees_at_slot = get_crosslink_committees_at_slot(state, slot)
+      for crosslink_committee in crosslink_committees_at_slot:
+        # TODO https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#crosslinks-1
+        # but this is a best guess based on reasonableness of what "index" is
+        for index in crosslink_committee.a:
+          if index in attesting_validators(crosslink_committee):
+            state.validator_balances[index.int] += base_reward(state, index) * total_attesting_balance(crosslink_committee) div total_balance_sac(crosslink_committee)
           else:
             # TODO underflows?
             state.validator_balances[index] -= base_reward(state, index)
 
+  block: # Ejections
+    process_ejections(state)
+
   block: # Validator registry
-    if state.finalized_slot > state.validator_registry_latest_change_slot and
-        state.shard_committees_at_slots.allIt(
-          it.allIt(
-            state.latest_crosslinks[it.shard].slot >
-              state.validator_registry_latest_change_slot)):
+    # https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#validator-registry
+    state.previous_epoch_calculation_slot = state.current_epoch_calculation_slot
+    state.previous_epoch_start_shard = state.current_epoch_start_shard
+    state.previous_epoch_randao_mix = state.current_epoch_randao_mix
+
+    if state.finalized_slot > state.validator_registry_update_slot and
+       allIt(
+         0 ..< get_current_epoch_committee_count_per_slot(state).int * EPOCH_LENGTH,
+         state.latest_crosslinks[(state.current_epoch_start_shard + it.uint64) mod SHARD_COUNT].slot > state.validator_registry_update_slot):
       update_validator_registry(state)
-
-      state.validator_registry_latest_change_slot = state.slot
-
-      for i in 0..<EPOCH_LENGTH:
-        state.shard_committees_at_slots[i] =
-          state.shard_committees_at_slots[EPOCH_LENGTH + i]
-
+      state.current_epoch_calculation_slot = state.slot
+      state.current_epoch_start_shard = (state.current_epoch_start_shard + get_current_epoch_committee_count_per_slot(state) * EPOCH_LENGTH) mod SHARD_COUNT
+      state.current_epoch_randao_mix = get_randao_mix(state, state.current_epoch_calculation_slot - SEED_LOOKAHEAD)
     else:
       # If a validator registry change does NOT happen
-      for i in 0..<EPOCH_LENGTH:
-        state.shard_committees_at_slots[i] =
-          state.shard_committees_at_slots[EPOCH_LENGTH + i]
-
-      let
-        epochs_since_last_registry_change =
-          (state.slot - state.validator_registry_latest_change_slot) div
-            EPOCH_LENGTH
-        start_shard = state.shard_committees_at_slots[0][0].shard
+      let epochs_since_last_registry_change = (state.slot - state.validator_registry_update_slot) div EPOCH_LENGTH
+      if is_power_of_2(epochs_since_last_registry_change):
+        state.current_epoch_calculation_slot = state.slot
+        state.current_epoch_randao_mix = get_randao_mix(state, state.current_epoch_calculation_slot - SEED_LOOKAHEAD)
+    # TODO run process_penalties_and_exits
 
   block: # Final updates
     state.latest_attestations.keepItIf(

research/serialized_sizes.nim

@@ -1,7 +1,7 @@
 import
   confutils,
   ../beacon_chain/[extras, ssz],
-  ../beacon_chain/spec/[beaconstate, datatypes, digest],
+  ../beacon_chain/spec/[beaconstate, datatypes, digest, validator],
   ../tests/testutil
 
 proc stateSize(deposits: int, maxContent = false) =
@@ -13,8 +13,10 @@ proc stateSize(deposits: int, maxContent = false) =
     #      of attestations, and each block has a cap on the number of
     #      attestations it may hold, so we'll just add so many of them
     state.latest_attestations.setLen(MAX_ATTESTATIONS * EPOCH_LENGTH * 2)
-    let validatorsPerCommittee =
-      len(state.shard_committees_at_slots[0][0].committee) # close enough..
+    let
+      crosslink_committees = get_crosslink_committees_at_slot(state, 0)
+      validatorsPerCommittee =
+        len(crosslink_committees[0].a) # close enough..
     for a in state.latest_attestations.mitems():
       a.participation_bitfield.setLen(validatorsPerCommittee)
   echo "Validators: ", deposits, ", total: ", state.serialize().len

research/state_sim.nim

@@ -4,7 +4,7 @@ import
   options, sequtils, random,
   milagro_crypto,
   ../tests/[testutil],
-  ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers],
+  ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator],
   ../beacon_chain/[extras, ssz, state_transition, fork_choice]
 
 proc `%`(v: uint64): JsonNode = newJInt(v.BiggestInt)

tests/test_state_transition.nim

@@ -8,7 +8,7 @@
 import
   options, sequtils, unittest,
   ./testutil,
-  ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers],
+  ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator],
   ../beacon_chain/[extras, state_transition, ssz]
 
 suite "Block processing":
@@ -101,9 +101,11 @@ suite "Block processing":
 
     let
       # Create an attestation for slot 1 signed by the only attester we have!
+      crosslink_committees = get_crosslink_committees_at_slot(state, state.slot)
       attestation = makeAttestation(
         state, previous_block_root,
-        state.shard_committees_at_slots[state.slot][0].committee[0])
+        #state.shard_committees_at_slots[state.slot][0].committee[0])
+        crosslink_committees[0].a[0])
 
     # Some time needs to pass before attestations are included - this is
     # to let the attestation propagate properly to interested participants