diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index f5f38de5eb..42845c235a 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -94,8 +94,8 @@ EPOCHS_PER_ETH1_VOTING_PERIOD: 32 SLOTS_PER_HISTORICAL_ROOT: 8192 # 2**8 (= 256) epochs ~27 hours MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 -# 2**11 (= 2,048) epochs 9 days -PERSISTENT_COMMITTEE_PERIOD: 2048 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 # 2**6 (= 64) epochs ~7 hours MAX_EPOCHS_PER_CROSSLINK: 64 # 2**2 (= 4) epochs 25.6 minutes @@ -122,8 +122,8 @@ BASE_REWARD_FACTOR: 64 WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 -# 2**25 (= 33,554,432) -INACTIVITY_PENALTY_QUOTIENT: 33554432 +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT: 16777216 # 2**5 (= 32) MIN_SLASHING_PENALTY_QUOTIENT: 32 @@ -132,8 +132,8 @@ MIN_SLASHING_PENALTY_QUOTIENT: 32 # --------------------------------------------------------------- # 2**4 (= 16) MAX_PROPOSER_SLASHINGS: 16 -# 2**0 (= 1) -MAX_ATTESTER_SLASHINGS: 1 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 # 2**7 (= 128) MAX_ATTESTATIONS: 128 # 2**4 (= 16) @@ -175,8 +175,6 @@ ONLINE_PERIOD: 8 LIGHT_CLIENT_COMMITTEE_SIZE: 128 # 2**8 (= 256) | epochs | ~27 hours LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**8 (= 256) | epochs | ~27 hours -SHARD_COMMITTEE_PERIOD: 256 # 2**18 (= 262,144) SHARD_BLOCK_CHUNK_SIZE: 262144 # 2**2 (= 4) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 7fc255f59d..d8e346ffab 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -89,13 +89,13 @@ MIN_SEED_LOOKAHEAD: 1 # 2**2 (= 4) epochs MAX_SEED_LOOKAHEAD: 4 # [customized] higher frequency new deposits from eth1 for testing -EPOCHS_PER_ETH1_VOTING_PERIOD: 2 +EPOCHS_PER_ETH1_VOTING_PERIOD: 4 # [customized] smaller state SLOTS_PER_HISTORICAL_ROOT: 64 # 2**8 (= 256) epochs MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 # [customized] higher frequency of committee turnover and faster time to acceptable voluntary exit -PERSISTENT_COMMITTEE_PERIOD: 128 +SHARD_COMMITTEE_PERIOD: 64 # [customized] fast catchup crosslinks MAX_EPOCHS_PER_CROSSLINK: 4 # 2**2 (= 4) epochs @@ -122,8 +122,8 @@ BASE_REWARD_FACTOR: 64 WHISTLEBLOWER_REWARD_QUOTIENT: 512 # 2**3 (= 8) PROPOSER_REWARD_QUOTIENT: 8 -# 2**25 (= 33,554,432) -INACTIVITY_PENALTY_QUOTIENT: 33554432 +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT: 16777216 # 2**5 (= 32) MIN_SLASHING_PENALTY_QUOTIENT: 32 @@ -132,8 +132,8 @@ MIN_SLASHING_PENALTY_QUOTIENT: 32 # --------------------------------------------------------------- # 2**4 (= 16) MAX_PROPOSER_SLASHINGS: 16 -# 2**0 (= 1) -MAX_ATTESTER_SLASHINGS: 1 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 # 2**7 (= 128) MAX_ATTESTATIONS: 128 # 2**4 (= 16) @@ -178,8 +178,6 @@ ONLINE_PERIOD: 8 LIGHT_CLIENT_COMMITTEE_SIZE: 128 # 2**8 (= 256) | epochs LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**8 (= 256) | epochs -SHARD_COMMITTEE_PERIOD: 256 # 2**18 (= 262,144) SHARD_BLOCK_CHUNK_SIZE: 262144 # 2**2 (= 4) diff --git a/setup.py b/setup.py index e0d6561dd6..5f0dce7631 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ def get_spec(file_name: str) -> SpecObject: PHASE1_IMPORTS = '''from eth2spec.phase0 import spec as phase0 from eth2spec.config.config_util import apply_constants_config from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional ) from dataclasses import ( @@ -146,8 +146,11 @@ def ceillog2(x: uint64) -> int: hash_cache: Dict[bytes, Bytes32] = {} -def get_eth1_data(distance: uint64) -> Bytes32: - return hash(distance) +def get_eth1_data(block: Eth1Block) -> Eth1Data: + """ + A stub function return mocking Eth1Data. + """ + return Eth1Data(block_hash=hash_tree_root(block)) def hash(x: bytes) -> Bytes32: # type: ignore @@ -373,6 +376,7 @@ def finalize_options(self): self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md + specs/phase0/validator.md specs/phase1/custody-game.md specs/phase1/beacon-chain.md specs/phase1/shard-transition.md @@ -497,7 +501,7 @@ def run(self): "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==2.0.0", + "py_ecc==4.0.0", "dataclasses==0.6", "remerkleable==0.1.13", "ruamel.yaml==0.16.5", diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 262cc6797c..142cf3b025 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -35,7 +35,7 @@ - [`DepositMessage`](#depositmessage) - [`DepositData`](#depositdata) - [`BeaconBlockHeader`](#beaconblockheader) - - [`SigningRoot`](#signingroot) + - [`SigningData`](#signingdata) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) - [`AttesterSlashing`](#attesterslashing) @@ -195,7 +195,6 @@ The following values are (non-configurable) constants used throughout the specif | `HYSTERESIS_DOWNWARD_MULTIPLIER` | `1` | | `HYSTERESIS_UPWARD_MULTIPLIER` | `5` | - - For the safety of committees, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](http://web.archive.org/web/20190504131341/https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Gwei values @@ -228,7 +227,8 @@ The following values are (non-configurable) constants used throughout the specif | `EPOCHS_PER_ETH1_VOTING_PERIOD` | `2**5` (= 32) | epochs | ~3.4 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~27 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | -| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | +| `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | + ### State list lengths @@ -246,17 +246,17 @@ The following values are (non-configurable) constants used throughout the specif | `BASE_REWARD_FACTOR` | `2**6` (= 64) | | `WHISTLEBLOWER_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | -| `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | +| `INACTIVITY_PENALTY_QUOTIENT` | `2**24` (= 16,777,216) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. +- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12` epochs (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block | Name | Value | | - | - | | `MAX_PROPOSER_SLASHINGS` | `2**4` (= 16) | -| `MAX_ATTESTER_SLASHINGS` | `2**0` (= 1) | +| `MAX_ATTESTER_SLASHINGS` | `2**1` (= 2) | | `MAX_ATTESTATIONS` | `2**7` (= 128) | | `MAX_DEPOSITS` | `2**4` (= 16) | | `MAX_VOLUNTARY_EXITS` | `2**4` (= 16) | @@ -273,7 +273,6 @@ The following values are (non-configurable) constants used throughout the specif | `DOMAIN_SELECTION_PROOF` | `DomainType('0x05000000')` | | `DOMAIN_AGGREGATE_AND_PROOF` | `DomainType('0x06000000')` | - ## Containers The following types are [SimpleSerialize (SSZ)](../../ssz/simple-serialize.md) containers. @@ -403,10 +402,10 @@ class BeaconBlockHeader(Container): body_root: Root ``` -#### `SigningRoot` +#### `SigningData` ```python -class SigningRoot(Container): +class SigningData(Container): object_root: Root domain: Domain ``` @@ -608,16 +607,18 @@ def bytes_to_int(data: bytes) -> uint64: #### BLS Signatures -Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-00). Specifically, eth2 uses the `BLS_SIG_BLS12381G2-SHA256-SSWU-RO-_POP_` ciphersuite which implements the following interfaces: +Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification draft-irtf-cfrg-bls-signature-02](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-02) but uses [Hashing to Elliptic Curves - draft-irtf-cfrg-hash-to-curve-07](https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-07) instead of draft-irtf-cfrg-hash-to-curve-06. Specifically, eth2 uses the `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` ciphersuite which implements the following interfaces: - `def Sign(SK: int, message: Bytes) -> BLSSignature` - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` - `def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` -- `def AggregateVerify(pairs: Sequence[PK: BLSPubkey, message: Bytes], signature: BLSSignature) -> bool` +- `def AggregateVerify(PKs: Sequence[BLSPubkey], messages: Sequence[Bytes], signature: BLSSignature) -> bool` Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. +*Note*: The non-standard configuration of the BLS and hash to curve specs is temporary and will be resolved once IETF releases BLS spec draft 3. + ### Predicates #### `is_active_validator` @@ -688,11 +689,11 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ - Check if ``indexed_attestation`` has sorted and unique indices and a valid aggregate signature. + Check if ``indexed_attestation`` is not empty, has sorted and unique indices and has a valid aggregate signature. """ # Verify indices are sorted and unique indices = indexed_attestation.attesting_indices - if not indices == sorted(set(indices)): + if len(indices) == 0 or not indices == sorted(set(indices)): return False # Verify aggregate signature pubkeys = [state.validators[i].pubkey for i in indices] @@ -722,9 +723,9 @@ def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint #### `compute_shuffled_index` ```python -def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Bytes32) -> ValidatorIndex: +def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> uint64: """ - Return the shuffled validator index corresponding to ``seed`` (and ``index_count``). + Return the shuffled index corresponding to ``seed`` (and ``index_count``). """ assert index < index_count @@ -732,14 +733,14 @@ def compute_shuffled_index(index: ValidatorIndex, index_count: uint64, seed: Byt # See the 'generalized domain' algorithm on page 3 for current_round in range(SHUFFLE_ROUND_COUNT): pivot = bytes_to_int(hash(seed + int_to_bytes(current_round, length=1))[0:8]) % index_count - flip = ValidatorIndex((pivot + index_count - index) % index_count) + flip = (pivot + index_count - index) % index_count position = max(index, flip) source = hash(seed + int_to_bytes(current_round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] bit = (byte >> (position % 8)) % 2 index = flip if bit else index - return ValidatorIndex(index) + return index ``` #### `compute_proposer_index` @@ -753,11 +754,11 @@ def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex] MAX_RANDOM_BYTE = 2**8 - 1 i = 0 while True: - candidate_index = indices[compute_shuffled_index(ValidatorIndex(i % len(indices)), len(indices), seed)] + candidate_index = indices[compute_shuffled_index(i % len(indices), len(indices), seed)] random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validators[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return ValidatorIndex(candidate_index) + return candidate_index i += 1 ``` @@ -773,7 +774,7 @@ def compute_committee(indices: Sequence[ValidatorIndex], """ start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[compute_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] + return [indices[compute_shuffled_index(i, len(indices), seed)] for i in range(start, end)] ``` #### `compute_epoch_at_slot` @@ -852,13 +853,12 @@ def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_ ```python def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: """ - Return the signing root of an object by calculating the root of the object-domain tree. + Return the signing root for the corresponding signing data. """ - domain_wrapped_object = SigningRoot( + return hash_tree_root(SigningData( object_root=hash_tree_root(ssz_object), domain=domain, - ) - return hash_tree_root(domain_wrapped_object) + )) ``` ### Beacon state accessors @@ -1125,7 +1125,7 @@ def slash_validator(state: BeaconState, whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) increase_balance(state, proposer_index, proposer_reward) - increase_balance(state, whistleblower_index, whistleblower_reward - proposer_reward) + increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) ``` ## Genesis @@ -1231,7 +1231,7 @@ def process_slots(state: BeaconState, slot: Slot) -> None: # Process epoch on the start slot of the next epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) - state.slot += Slot(1) + state.slot = Slot(state.slot + 1) ``` ```python @@ -1781,7 +1781,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Exits must specify an epoch when they become valid; they are not valid before then assert get_current_epoch(state) >= voluntary_exit.epoch # Verify the validator has been active long enough - assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD + assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD # Verify signature domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) signing_root = compute_signing_root(voluntary_exit, domain) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 60d398dd58..4ed2733e1a 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -162,7 +162,7 @@ def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices - if (i in store.latest_messages + if (i in store.latest_messages and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) )) ``` @@ -285,6 +285,10 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: # Attestations must not be for blocks in the future. If not, the attestation should not be considered assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot + # FFG and LMD vote must be consistent with each other + target_slot = compute_start_slot_at_epoch(target.epoch) + assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) + # Attestations can only affect the fork choice of subsequent slots. # Delay consideration in the fork choice until their slot is in the past. assert get_current_slot(store) >= attestation.data.slot + 1 diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index f4f2f3d229..0e8699555d 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -4,7 +4,7 @@ This document contains the networking specification for Ethereum 2.0 clients. It consists of four main sections: -1. A specification of the network fundamentals detailing the two network configurations: interoperability test network and mainnet launch. +1. A specification of the network fundamentals. 2. A specification of the three network interaction *domains* of Eth2: (a) the gossip domain, (b) the discovery domain, and (c) the Req/Resp domain. 3. The rationale and further explanation for the design choices made in the previous two sections. 4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. @@ -17,14 +17,8 @@ It consists of four main sections: - [Network fundamentals](#network-fundamentals) - [Transport](#transport) - - [Interop](#interop) - - [Mainnet](#mainnet) - [Encryption and identification](#encryption-and-identification) - - [Interop](#interop-1) - - [Mainnet](#mainnet-1) - [Protocol Negotiation](#protocol-negotiation) - - [Interop](#interop-2) - - [Mainnet](#mainnet-2) - [Multiplexing](#multiplexing) - [Eth2 network interaction domains](#eth2-network-interaction-domains) - [Configuration](#configuration) @@ -33,11 +27,8 @@ It consists of four main sections: - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) - [Attestation subnets](#attestation-subnets) - - [Interop](#interop-3) - - [Mainnet](#mainnet-3) + - [Attestations and Aggregation](#attestations-and-aggregation) - [Encodings](#encodings) - - [Interop](#interop-4) - - [Mainnet](#mainnet-4) - [The Req/Resp domain](#the-reqresp-domain) - [Protocol identification](#protocol-identification) - [Req/Resp interaction](#reqresp-interaction) @@ -56,29 +47,25 @@ It consists of four main sections: - [Integration into libp2p stacks](#integration-into-libp2p-stacks) - [ENR structure](#enr-structure) - [Attestation subnet bitfield](#attestation-subnet-bitfield) - - [Interop](#interop-5) - - [Mainnet](#mainnet-5) - - [`eth2` field](#eth2-field) - - [General capabilities](#general-capabilities) + - [`eth2` field](#eth2-field) + - [General capabilities](#general-capabilities) - [Topic advertisement](#topic-advertisement) - - [Mainnet](#mainnet-6) - [Design decision rationale](#design-decision-rationale) - [Transport](#transport-1) - [Why are we defining specific transports?](#why-are-we-defining-specific-transports) - [Can clients support other transports/handshakes than the ones mandated by the spec?](#can-clients-support-other-transportshandshakes-than-the-ones-mandated-by-the-spec) - [What are the advantages of using TCP/QUIC/Websockets?](#what-are-the-advantages-of-using-tcpquicwebsockets) - [Why do we not just support a single transport?](#why-do-we-not-just-support-a-single-transport) - - [Why are we not using QUIC for mainnet from the start?](#why-are-we-not-using-quic-for-mainnet-from-the-start) + - [Why are we not using QUIC from the start?](#why-are-we-not-using-quic-from-the-start) - [Multiplexing](#multiplexing-1) - [Why are we using mplex/yamux?](#why-are-we-using-mplexyamux) - [Protocol Negotiation](#protocol-negotiation-1) - - [When is multiselect 2.0 due and why are we using it for mainnet?](#when-is-multiselect-20-due-and-why-are-we-using-it-for-mainnet) + - [When is multiselect 2.0 due and why do we plan to migrate to it?](#when-is-multiselect-20-due-and-why-do-we-plan-to-migrate-to-it) - [What is the difference between connection-level and stream-level protocol negotiation?](#what-is-the-difference-between-connection-level-and-stream-level-protocol-negotiation) - [Encryption](#encryption) - - [Why are we using SecIO for interop? Why not for mainnet?](#why-are-we-using-secio-for-interop-why-not-for-mainnet) - - [Why are we using Noise/TLS 1.3 for mainnet?](#why-are-we-using-noisetls-13-for-mainnet) + - [Why are we not supporting SecIO?](#why-are-we-not-supporting-secio) + - [Why are we using Noise/TLS 1.3?](#why-are-we-using-noisetls-13) - [Why are we using encryption at all?](#why-are-we-using-encryption-at-all) - - [Will mainnnet networking be untested when it launches?](#will-mainnnet-networking-be-untested-when-it-launches) - [Gossipsub](#gossipsub) - [Why are we using a pub/sub algorithm for block and attestation propagation?](#why-are-we-using-a-pubsub-algorithm-for-block-and-attestation-propagation) - [Why are we using topics to segregate encodings, yet only support one encoding?](#why-are-we-using-topics-to-segregate-encodings-yet-only-support-one-encoding) @@ -111,6 +98,7 @@ It consists of four main sections: - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers) - [Why are using Snappy for compression?](#why-are-using-snappy-for-compression) - [Can I get access to unencrypted bytes on the wire for debugging purposes?](#can-i-get-access-to-unencrypted-bytes-on-the-wire-for-debugging-purposes) + - [What are SSZ type size bounds?](#what-are-ssz-type-size-bounds) - [libp2p implementations matrix](#libp2p-implementations-matrix) @@ -120,46 +108,22 @@ It consists of four main sections: This section outlines the specification for the networking stack in Ethereum 2.0 clients. -Sections that have differing parameters for mainnet launch and interoperability testing are split into subsections. Sections that are not split have the same parameters for interoperability testing as mainnet launch. - ## Transport Even though libp2p is a multi-transport stack (designed to listen on multiple simultaneous transports and endpoints transparently), we hereby define a profile for basic interoperability. -#### Interop - All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). -To facilitate connectivity and avert possible IPv6 routability/support issues, clients participating in the interoperability testnet MUST expose at least ONE IPv4 endpoint. +Clients must support listening on at least one of IPv4 or IPv6. Clients that do _not_ have support for listening on IPv4 SHOULD be cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST be capable of dialing both IPv4 and IPv6 addresses. -All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. +All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. (Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined soon.) Nodes operating behind a NAT, or otherwise undialable by default (e.g. container runtime, firewall, etc.), MUST have their infrastructure configured to enable inbound traffic on the announced public listening endpoint. -#### Mainnet - -All requirements from the interoperability testnet apply, except for the IPv4 addressing scheme requirement. - -At this stage, clients are licensed to drop IPv4 support if they wish to do so, cognizant of the potential disadvantages in terms of Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST retain capability to dial both IPv4 and IPv6 addresses. - -Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined closer to the time. - ## Encryption and identification -#### Interop - -[SecIO](https://github.com/libp2p/specs/tree/master/secio) with `secp256k1` identities will be used for initial interoperability testing. - -The following SecIO parameters MUST be supported by all stacks: - -- Key agreement: ECDH-P256. -- Cipher: AES-128. -- Digest: SHA-256. - -#### Mainnet - The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure -channel handshake with `secp256k1` identities will be used for mainnet. +channel handshake with `secp256k1` identities will be used for encryption. As specified in the libp2p specification, clients MUST support the `XX` handshake pattern. @@ -167,13 +131,7 @@ As specified in the libp2p specification, clients MUST support the `XX` handshak Clients MUST use exact equality when negotiating protocol versions to use and MAY use the version to give priority to higher version numbers. -#### Interop - -Connection-level and stream-level (see the [Rationale](#design-decision-rationale) section below for explanations) protocol negotiation MUST be conducted using [multistream-select v1.0](https://github.com/multiformats/multistream-select/). Its protocol ID is: `/multistream/1.0.0`. - -#### Mainnet - -Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95). Depending on the number of clients that have implementations for multiselect 2.0 by mainnet, [multistream-select 1.0](https://github.com/multiformats/multistream-select/) may be phased out. +Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95) when the spec solidifies. Once all clients have implementations for multiselect 2.0, multistream-select 1.0 MAY be phased out. ## Multiplexing @@ -181,7 +139,7 @@ During connection bootstrapping, libp2p dynamically negotiates a mutually suppor Two multiplexers are commonplace in libp2p implementations: [mplex](https://github.com/libp2p/specs/tree/master/mplex) and [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). Their protocol IDs are, respectively: `/mplex/6.7.0` and `/yamux/1.0.0`. -Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux must take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. +Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux MUST take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. # Eth2 network interaction domains @@ -193,7 +151,6 @@ This section outlines constants that are used in this spec. |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | -| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | | `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | @@ -219,9 +176,9 @@ Where ## The gossip domain: gossipsub -Clients MUST support the [gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) libp2p protocol. +Clients MUST support the [gossipsub v1](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) libp2p protocol including the [gossipsub v1.1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md) extension. -**Protocol ID:** `/meshsub/1.0.0` +**Protocol ID:** `/meshsub/1.1.0` **Gossipsub Parameters** @@ -261,77 +218,74 @@ where `base64` is the [URL-safe base64 alphabet](https://tools.ietf.org/html/rfc The payload is carried in the `data` field of a gossipsub message, and varies depending on the topic: -| Name | Message Type | -|------------------------------------------------|-------------------------| -| beacon_block | SignedBeaconBlock | -| beacon_aggregate_and_proof | SignedAggregateAndProof | -| beacon_attestation\* | Attestation | -| committee_index{subnet_id}\_beacon_attestation | Attestation | -| voluntary_exit | SignedVoluntaryExit | -| proposer_slashing | ProposerSlashing | -| attester_slashing | AttesterSlashing | +| Name | Message Type | +|----------------------------------|---------------------------| +| `beacon_block` | `SignedBeaconBlock` | +| `beacon_aggregate_and_proof` | `SignedAggregateAndProof` | +| `beacon_attestation_{subnet_id}` | `Attestation` | +| `voluntary_exit` | `SignedVoluntaryExit` | +| `proposer_slashing` | `ProposerSlashing` | +| `attester_slashing` | `AttesterSlashing` | Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. When processing incoming gossip, clients MAY descore or disconnect peers who fail to observe these constraints. -\* The `beacon_attestation` topic is only for interop and will be removed prior to mainnet. +Gossipsub v1.1 introduces [Extended Validators](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#extended-validators) for the application to aid in the gossipsub peer-scoring scheme. +We utilize `ACCEPT`, `REJECT`, and `IGNORE`. For each gossipsub topic, there are application specific validations. If all validations pass, return `ACCEPT`. If one or more validations fail while processing the items in order, return either `REJECT` or `IGNORE` as specified in the prefix of the particular condition. #### Global topics There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `Name`s are: - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network - - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - - The block is from a slot greater than the latest finalized slot -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). - - The block is the first block with valid signature received for the proposer for the slot, `signed_beacon_block.message.slot`. - - The proposer signature, `signed_beacon_block.signature`, is valid with respect to the `proposer_index` pubkey. - - The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the block MAY be queued for later processing while proposers for the block's branch are calculated. + - _[IGNORE]_ The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). + - _[IGNORE]_ The block is from a slot greater than the latest finalized slot -- i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). + - _[IGNORE]_ The block is the first block with valid signature received for the proposer for the slot, `signed_beacon_block.message.slot`. + - _[REJECT]_ The proposer signature, `signed_beacon_block.signature`, is valid with respect to the `proposer_index` pubkey. + - _[REJECT]_ The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, the block MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) - - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). - - The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). - - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. - - The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`. - - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. - - The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. - - The signature of `aggregate` is valid. + - _[IGNORE]_ `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot). + - _[IGNORE]_ The valid aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally). + - _[IGNORE]_ The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. + - _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. + - _[REJECT]_ The attestation has participants -- that is, `len(get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)) >= 1`. + - _[REJECT]_ `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. + - _[REJECT]_ The aggregator's validator index is within the committee -- i.e. `aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, aggregate.data.index)`. + - _[REJECT]_ The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. + - _[REJECT]_ The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. + - _[REJECT]_ The signature of `aggregate` is valid. Additional global topics are used to propagate lower frequency validator messages. Their `Name`s are: - `voluntary_exit` - This topic is used solely for propagating signed voluntary validator exits to proposers on the network. Signed voluntary exits are sent in their entirety. The following validations MUST pass before forwarding the `signed_voluntary_exit` on to the network - - The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - - All of the conditions within `process_voluntary_exit` pass validation. + - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. + - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. - - All of the conditions within `process_proposer_slashing` pass validation. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - - At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). - - All of the conditions within `process_attester_slashing` pass validation. + - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). + - _[REJECT]_ All of the conditions within `process_attester_slashing` pass validation. #### Attestation subnets Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: -- `committee_index{subnet_id}_beacon_attestation` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - - There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. - - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - - The signature of `attestation` is valid. - -#### Interop +- `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. + - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`). + - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). + - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`). + - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. + - _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. + - _[REJECT]_ The signature of `attestation` is valid. -Unaggregated and aggregated attestations from all shards are sent as `Attestation`s to the `beacon_attestation` topic. Clients are not required to publish aggregate attestations but must be able to process them. All validating clients SHOULD try to perform local attestation aggregation to prepare for block proposing. +#### Attestations and Aggregation -#### Mainnet +Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1. -Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. For the `committee_index{subnet_id}_beacon_attestation` topics, `subnet_id` is set to `index % ATTESTATION_SUBNET_COUNT`, where `index` is the `CommitteeIndex` of the given committee. - -Unaggregated attestations are sent to the subnet topic, `committee_index{attestation.data.index % ATTESTATION_SUBNET_COUNT}_beacon_attestation` as `Attestation`s. +Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s. Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. @@ -339,17 +293,11 @@ Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `A Topics are post-fixed with an encoding. Encodings define how the payload of a gossipsub message is encoded. -#### Interop - -- `ssz` - All objects are [SSZ-encoded](#ssz-encoding). Example: The beacon block topic string is `/eth2/beacon_block/ssz`, and the data field of a gossipsub message is an ssz-encoded `SignedBeaconBlock`. - -#### Mainnet - - `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. Example: The beacon aggregate attestation topic string is `/eth2/446a7232/beacon_aggregate_and_proof/ssz_snappy`, the fork digest is `446a7232` and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy. Snappy has two formats: "block" and "frames" (streaming). Gossip messages remain relatively small (100s of bytes to 100s of kilobytes) so [basic snappy block compression](https://github.com/google/snappy/blob/master/format_description.txt) is used to avoid the additional overhead associated with snappy frames. -Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations. +Implementations MUST use a single encoding for gossip. Changing an encoding will require coordination between participating implementations. ## The Req/Resp domain @@ -385,9 +333,12 @@ result ::= “0” | “1” | “2” | [“128” ... ”255”] The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. Because req/resp streams are single-use and stream closures implicitly delimit the boundaries, it is not strictly necessary to length-prefix payloads; however, certain encodings like SSZ do, for added security. -A `response` is formed by zero or more `response_chunk`s. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. The encoded-payload of a `response_chunk` has a maximum uncompressed byte size of `MAX_CHUNK_SIZE`. +A `response` is formed by zero or more `response_chunk`s. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. + +For both `request`s and `response`s, the `encoding-dependent-header` MUST be valid, and the `encoded-payload` must be valid within the constraints of the `encoding-dependent-header`. +This includes type-specific bounds on payload size for some encoding strategies. Regardless of these type specific bounds, a global maximum uncompressed byte size of `MAX_CHUNK_SIZE` MUST be applied to all method response chunks. -Clients MUST ensure the each encoded payload of a `response_chunk` is less than or equal to `MAX_CHUNK_SIZE`; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. +Clients MUST ensure that lengths are within these bounds; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. #### Requesting side @@ -395,13 +346,22 @@ Once a new stream with the protocol ID for the request type has been negotiated, The requester MUST close the write side of the stream once it finishes writing the request message. At this point, the stream will be half-closed. -The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. For responses consisting of potentially many `response_chunk`s (an SSZ-list) the requester SHOULD read from the stream until either; a) An error result is received in one of the chunks, b) The responder closes the stream, c) More than `MAX_CHUNK_SIZE` bytes have been read for a single `response_chunk` payload or d) More than the maximum number of requested chunks are read. For requests consisting of a single `response_chunk` and a length-prefix, the requester should read the exact number of bytes defined by the length-prefix before closing the stream. +The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. +A requester SHOULD read from the stream until either: +1. An error result is received in one of the chunks (the error payload MAY be read before stopping). +2. The responder closes the stream. +3. Any part of the `response_chunk` fails validation. +4. The maximum number of requested chunks are read. + +For requests consisting of a single valid `response_chunk`, the requester SHOULD read the chunk fully, as defined by the `encoding-dependent-header`, before closing the stream. + #### Responding side -Once a new stream with the protocol ID for the request type has been negotiated, the responder must process the incoming request message according to the encoding strategy, until EOF (denoting stream half-closure by the requester). +Once a new stream with the protocol ID for the request type has been negotiated, the responder SHOULD process the incoming request and MUST validate it before processing it. +Request processing and validation MUST be done according to the encoding strategy, until EOF (denoting stream half-closure by the requester). The responder MUST: @@ -437,20 +397,12 @@ The `ErrorMessage` schema is: *Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. As the `ErrorMessage` is not an SSZ-container, only the UTF-8 bytes will be sent when SSZ-encoded. -A response therefore has the form of one or more `response_chunk`s, each structured as follows: -``` - +--------+--------+--------+--------+--------+--------+ - | result | header (opt) | encoded_response | - +--------+--------+--------+--------+--------+--------+ -``` -Here, `result` represents the 1-byte response code. - ### Encoding strategies The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: - `ssz`: the contents are [SSZ-encoded](../../ssz/simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Root`'s. -- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. MAY be supported in the interoperability testnet; MUST be supported in mainnet. +- `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. This encoding type MUST be supported by all clients. #### SSZ-encoding strategy (with or without Snappy) @@ -471,18 +423,21 @@ If Snappy is applied, it can be passed through a buffered Snappy writer to compr *Reading*: After reading the expected SSZ byte length, the SSZ decoder can directly read the contents from the stream. If snappy is applied, it can be passed through a buffered Snappy reader to decompress frame by frame. -A reader SHOULD NOT read more than `max_encoded_len(n)` bytes after reading the SSZ length prefix `n` from the header. +Before reading the payload, the header MUST be validated: +- The unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes, which is sufficient for any `uint64`. +- The length-prefix is within the expected [size bounds derived from the payload SSZ type](#what-are-ssz-type-size-bounds). + +After reading a valid header, the payload MAY be read, while maintaining the size constraints from the header. + +A reader SHOULD NOT read more than `max_encoded_len(n)` bytes after reading the SSZ length-prefix `n` from the header. - For `ssz` this is: `n` - For `ssz_snappy` this is: `32 + n + n // 6`. This is considered the [worst-case compression result](https://github.com/google/snappy/blob/537f4ad6240e586970fe554614542e9717df7902/snappy.cc#L98) by Snappy. A reader SHOULD consider the following cases as invalid input: -- A SSZ length prefix that, compared against the SSZ type information (vector lengths, list limits, integer sizes, etc.), is: - - Smaller than the expected minimum serialized length. - - Bigger than the expected maximum serialized length. -- Any remaining bytes, after having read the `n` SSZ bytes. An EOF is expected. -- An early EOF, before fully reading the declared length prefix worth of SSZ bytes. +- Any remaining bytes, after having read the `n` SSZ bytes. An EOF is expected if more bytes are read than required. +- An early EOF, before fully reading the declared length-prefix worth of SSZ bytes. -In case of an invalid input, a reader MUST: +In case of an invalid input (header or payload), a reader MUST: - From requests: send back an error message, response code `InvalidRequest`. The request itself is ignored. - From responses: ignore the response, the response MUST be considered bad server behavior. @@ -593,7 +548,9 @@ The response MUST contain no more than `count` blocks. Clients MUST order blocks by increasing slot number. -Clients MUST respond with blocks from their view of the current fork choice. In particular, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. +Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. + +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. #### BeaconBlocksByRoot @@ -683,7 +640,7 @@ The response MUST consist of a single `response_chunk`. ## The discovery domain: discv5 -Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery, both in the interoperability testnet and mainnet. +Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) is used for peer discovery. `discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. `discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. @@ -723,15 +680,7 @@ If a node's `MetaData.attnets` has any non-zero bit, the ENR MUST include the `a If a node's `MetaData.attnets` is composed of all zeros, the ENR MAY optionally include the `attnets` entry or leave it out entirely. -#### Interop - -In the interoperability testnet, all peers will support all capabilities defined in this document (gossip, full Req/Resp suite, discovery protocol), therefore the ENR record does not need to carry Eth2 capability information, as it would be superfluous. - -Nonetheless, ENRs MUST carry a generic `eth2` key with nil value, denoting that the peer is indeed an Eth2 peer, in order to eschew connecting to Eth 1.0 peers. - -#### Mainnet - -##### `eth2` field +#### `eth2` field ENRs MUST carry a generic `eth2` key with an 16-byte value of the node's current fork digest, next fork version, and next fork epoch to ensure connections are made with peers on the intended eth2 network. @@ -763,14 +712,12 @@ Clients SHOULD connect to peers with `fork_digest`, `next_fork_version`, and `ne Clients MAY connect to peers with the same `fork_digest` but a different `next_fork_version`/`next_fork_epoch`. Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. -##### General capabilities +#### General capabilities -On mainnet, ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. +ENRs MUST include a structure enumerating the capabilities offered by the peer in an efficient manner. The concrete solution is currently undefined. Proposals include using namespaced bloom filters mapping capabilities to specific protocol IDs supported under that capability. ### Topic advertisement -#### Mainnet - discv5's topic advertisement feature is not expected to be ready for mainnet launch of Phase 0. Once this feature is built out and stable, we expect to use topic advertisement as a rendezvous facility for peers on shards. Until then, the ENR [attestation subnet bitfield](#attestation-subnet-bitfield) will be used for discovery of peers on particular subnets. @@ -816,7 +763,7 @@ Modeling for upgradeability and dynamic transport selection from the get-go lays Clients can adopt new transports without breaking old ones, and the multi-transport ability enables constrained and sandboxed environments (e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), without the need for proxying or trust delegation to servers. -### Why are we not using QUIC for mainnet from the start? +### Why are we not using QUIC from the start? The QUIC standard is still not finalized (at working draft 22 at the time of writing), and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). One remarkable example is node.js, where the QUIC implementation is [in early development](https://github.com/nodejs/quic). @@ -832,13 +779,13 @@ Overlay multiplexers are not necessary with QUIC since the protocol provides nat ## Protocol Negotiation -### When is multiselect 2.0 due and why are we using it for mainnet? +### When is multiselect 2.0 due and why do we plan to migrate to it? multiselect 2.0 is currently being conceptualized. The debate started [on this issue](https://github.com/libp2p/specs/pull/95), but it got overloaded—as it tends to happen with large conceptual OSS discussions that touch the heart and core of a system. -In the following weeks (August 2019), there will be a renewed initiative to first define the requirements, constraints, assumptions, and features, in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. +At some point in 2020, we expect a renewed initiative to first define the requirements, constraints, assumptions, and features, in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. -We plan to use multiselect 2.0 for mainnet because it will: +We plan to eventually migrate to multiselect 2.0 because it will: 1. Reduce round trips during connection bootstrapping and stream protocol negotiation. 2. Enable efficient one-stream-per-request interaction patterns. @@ -860,17 +807,15 @@ At present, multistream-select 1.0 is used for both types of negotiation, but mu ## Encryption -### Why are we using SecIO for interop? Why not for mainnet? +### Why are we not supporting SecIO? SecIO has been the default encryption layer for libp2p for years. It is used in IPFS and Filecoin. And although it will be superseded shortly, it is proven to work at scale. -SecIO is the common denominator across the various language libraries at this stage. It is widely implemented. That’s why we have chosen to use it for initial interop to minimize overhead in getting to a basic interoperability testnet. - -We won’t be using it for mainnet because, amongst other things, it requires several round trips to be sound, and doesn’t support early data (0-RTT data), a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. +Although SecIO has wide language support, we won’t be using it for mainnet because, amongst other things, it requires several round trips to be sound, and doesn’t support early data (0-RTT data), a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. SecIO is not considered secure for the purposes of this spec. -### Why are we using Noise/TLS 1.3 for mainnet? +### Why are we using Noise/TLS 1.3? Copied from the Noise Protocol Framework [website](http://www.noiseprotocol.org): @@ -896,10 +841,6 @@ Transport level encryption secures message exchange and provides properties that Note that transport-level encryption is not exclusive of application-level encryption or cryptography. Transport-level encryption secures the communication itself, while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.). -### Will mainnnet networking be untested when it launches? - -Before launching mainnet, the testnet will be switched over to mainnet networking parameters, including Noise handshakes, and other new protocols. This gives us an opportunity to drill coordinated network upgrades and verifying that there are no significant upgradeability gaps. - ## Gossipsub ### Why are we using a pub/sub algorithm for block and attestation propagation? @@ -1008,7 +949,7 @@ Requests are segregated by protocol ID to: 2. Affording this level of granularity with a top-level protocol would imply creating as many variants (e.g. /protocol/43-{a,b,c,d,...}) as the cartesian product of RFCs inflight, O(n^2). 7. Allow us to simplify the payload of requests. Request-id’s and method-ids no longer need to be sent. The encoding/request type and version can all be handled by the framework. -**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to hinder interop testing. More info is to be released from the libp2p community in the coming weeks. +**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. It is somewhat naïve and introduces overhead on every request when negotiating streams, although implementation-specific optimizations are possible to save this cost. Multiselect 2.0 will eventually remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to significantly hinder this domain. ### Why are messages length-prefixed with a protobuf varint in the SSZ-encoding? @@ -1127,7 +1068,7 @@ For all these reasons, generically negotiating compression algorithms may be tre At this stage, the wisest choice is to consider libp2p a messenger of bytes, and to make application layer participate in compressing those bytes. This looks different depending on the interaction layer: -- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload (when dictated by the encoding token in the topic name) prior to publishing the message via the API. No length prefixing is necessary because protobuf takes care of bounding the field in the serialized form. +- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload (when dictated by the encoding token in the topic name) prior to publishing the message via the API. No length-prefixing is necessary because protobuf takes care of bounding the field in the serialized form. - Req/Resp domain: since we define custom protocols that operate on byte streams, implementers are encouraged to encapsulate the encoding and compression logic behind MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. ### Why are using Snappy for compression? @@ -1142,6 +1083,14 @@ If your libp2p library relies on frameworks/runtimes such as Netty (jvm) or Node For specific ad-hoc testing scenarios, you can use the [plaintext/2.0.0 secure channel](https://github.com/libp2p/specs/blob/master/plaintext/README.md) (which is essentially no-op encryption or message authentication), in combination with tcpdump or Wireshark to inspect the wire. +### What are SSZ type size bounds? + +The SSZ encoding outputs of each type have size bounds: each dynamic type, such as a list, has a "limit", which can be used to compute the maximum valid output size. +Note that for some more complex dynamic-length objects, element offsets (4 bytes each) may need to be included. +Other types are static, they have a fixed size: no dynamic-length content is involved, and the minimum and maximum bounds are the same. + +For reference, the type bounds can be computed ahead of time, [as per this example](https://gist.github.com/protolambda/db75c7faa1e94f2464787a480e5d613e). It is advisable to derive these lengths from the SSZ type definitions in use, to ensure that version changes do not cause out-of-sync type bounds. + # libp2p implementations matrix This section will soon contain a matrix showing the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 69be712796..adf23c8408 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -90,6 +90,7 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | | `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | | `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | +| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | ## Becoming a validator @@ -281,8 +282,8 @@ def voting_period_start_time(state: BeaconState) -> uint64: ```python def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: return ( - block.timestamp <= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE - and block.timestamp >= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 + block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start + and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start ) ``` @@ -340,9 +341,10 @@ It is useful to be able to run a state transition function (working on a copy of ```python def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: - process_slots(state, block.slot) - process_block(state, block) - return hash_tree_root(state) + temp_state: BeaconState = state.copy() + signed_block = SignedBeaconBlock(message=block) + temp_state = state_transition(temp_state, signed_block, validate_result=False) + return hash_tree_root(temp_state) ``` ##### Signature @@ -350,9 +352,9 @@ def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: `signed_block = SignedBeaconBlock(message=block, signature=block_signature)`, where `block_signature` is obtained from: ```python -def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(header.slot)) - signing_root = compute_signing_root(header, domain) +def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(block, domain) return bls.Sign(privkey, signing_root) ``` @@ -417,7 +419,19 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD #### Broadcast attestation -Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `committee_index{attestation.data.index % ATTESTATION_SUBNET_COUNT}_beacon_attestation` pubsub topic. +Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic. + +```python +def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: + """ + Compute the correct subnet for an attestation for Phase 0. + Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet. + """ + slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH + committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start + + return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT +``` ### Attestation aggregation @@ -445,7 +459,7 @@ def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_si If the validator is selected to aggregate (`is_aggregator()`), they construct an aggregate attestation via the following. -Collect `attestations` seen via gossip during the `slot` that have an equivalent `attestation_data` to that constructed by the validator, and create an `aggregate_attestation: Attestation` with the following fields. +Collect `attestations` seen via gossip during the `slot` that have an equivalent `attestation_data` to that constructed by the validator. If `len(attestations) > 0`, create an `aggregate_attestation: Attestation` with the following fields. ##### Data @@ -518,7 +532,7 @@ class SignedAggregateAndProof(Container): ## Phase 0 attestation subnet stability -Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`committee_index{subnet_id}_beacon_attestation`). To provide this stability, each validator must: +Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each validator must: * Randomly select and remain subscribed to `RANDOM_SUBNETS_PER_VALIDATOR` attestation subnets * Maintain advertisement of the randomly selected subnets in their node's ENR `attnets` entry by setting the randomly selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 03794fa2a8..3bb01e2629 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -55,6 +55,8 @@ - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - [`is_shard_attestation`](#is_shard_attestation) - [`is_winning_attestation`](#is_winning_attestation) + - [`optional_aggregate_verify`](#optional_aggregate_verify) + - [`optional_fast_aggregate_verify`](#optional_fast_aggregate_verify) - [Block processing](#block-processing) - [Operations](#operations) - [New Attestation processing](#new-attestation-processing) @@ -100,7 +102,6 @@ Configuration is not namespaced. Instead it is strictly an extension; | `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | | `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | | `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | | `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | @@ -111,6 +112,7 @@ Configuration is not namespaced. Instead it is strictly an extension; | `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | | `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | +| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | | ## Updated containers @@ -572,7 +574,8 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe attestation = indexed_attestation.attestation domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) aggregation_bits = attestation.aggregation_bits - assert len(aggregation_bits) == len(indexed_attestation.committee) + if not any(aggregation_bits) or len(aggregation_bits) != len(indexed_attestation.committee): + return False if len(attestation.custody_bits_blocks) == 0: # fall back on phase0 behavior if there is no shard data. @@ -596,7 +599,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe all_signing_roots.append(compute_signing_root(attestation_wrapper, domain)) else: assert not cbit - return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature) + return bls.AggregateVerify(all_pubkeys, all_signing_roots, signature=attestation.signature) ``` #### `is_shard_attestation` @@ -633,6 +636,36 @@ def is_winning_attestation(state: BeaconState, ) ``` +#### `optional_aggregate_verify` + +```python +def optional_aggregate_verify(pubkeys: Sequence[BLSPubkey], + messages: Sequence[Bytes32], + signature: BLSSignature) -> bool: + """ + If ``pubkeys`` is an empty list, the given ``signature`` should be a stub ``NO_SIGNATURE``. + Otherwise, verify it with standard BLS AggregateVerify API. + """ + if len(pubkeys) == 0: + return signature == NO_SIGNATURE + else: + return bls.AggregateVerify(pubkeys, messages, signature) +``` + +#### `optional_fast_aggregate_verify` + +```python +def optional_fast_aggregate_verify(pubkeys: Sequence[BLSPubkey], message: Bytes32, signature: BLSSignature) -> bool: + """ + If ``pubkeys`` is an empty list, the given ``signature`` should be a stub ``NO_SIGNATURE``. + Otherwise, verify it with standard BLS FastAggregateVerify API. + """ + if len(pubkeys) == 0: + return signature == NO_SIGNATURE + else: + return bls.FastAggregateVerify(pubkeys, message, signature) +``` + ### Block processing ```python @@ -764,7 +797,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr for header in headers ] # Verify combined proposer signature - assert bls.AggregateVerify(zip(pubkeys, signing_roots), signature=transition.proposer_signature_aggregate) + assert optional_aggregate_verify(pubkeys, signing_roots, transition.proposer_signature_aggregate) # Save updated state state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] @@ -942,12 +975,7 @@ def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockB slot = compute_previous_slot(state.slot) signing_root = compute_signing_root(get_block_root_at_slot(state, slot), get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(slot))) - if len(signer_pubkeys) == 0: - # TODO: handle the empty light_client_signature case? - assert block_body.light_client_signature == BLSSignature() - return - else: - assert bls.FastAggregateVerify(signer_pubkeys, signing_root, signature=block_body.light_client_signature) + assert optional_fast_aggregate_verify(signer_pubkeys, signing_root, block_body.light_client_signature) ``` ### Epoch transition diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index eb243f8fb8..5f5acd84fe 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -300,7 +300,7 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived domain = get_domain(state, DOMAIN_RANDAO, reveal.epoch) signing_roots = [compute_signing_root(root, domain) for root in [hash_tree_root(reveal.epoch), reveal.mask]] - assert bls.AggregateVerify(zip(pubkeys, signing_roots), reveal.reveal) + assert bls.AggregateVerify(pubkeys, signing_roots, reveal.reveal) if reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: # Full slashing when the secret was revealed so early it may be a valid custody diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index bc29e2dfe0..55b867faa9 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -278,10 +278,13 @@ def get_shard_transition(beacon_state: BeaconState, proposer_signatures = [] for proposal in proposals: shard_block_lengths.append(len(proposal.message.body)) - if proposal.signature != BLSSignature(): + if proposal.signature != NO_SIGNATURE: proposer_signatures.append(proposal.signature) - proposer_signature_aggregate = bls.Aggregate(proposer_signatures) + if len(proposer_signatures) > 0: + proposer_signature_aggregate = bls.Aggregate(proposer_signatures) + else: + proposer_signature_aggregate = NO_SIGNATURE return ShardTransition( start_slot=start_slot, diff --git a/ssz/simple-serialize.md b/ssz/simple-serialize.md index 1c4f588eb8..b8a6bc9a20 100644 --- a/ssz/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -211,8 +211,8 @@ We first define helper functions: * `List[B, N]` and `Vector[B, N]`, where `B` is a basic type: `(N * size_of(B) + 31) // 32` (dividing by chunk size, rounding up) * `List[C, N]` and `Vector[C, N]`, where `C` is a composite type: `N` * containers: `len(fields)` -* `bitfield_bytes(bits)`: return the bits of the bitlist or bitvector, packed in bytes, aligned to the start. Length-delimiting bit for bitlists is excluded. -* `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. +* `pack(value)`: given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. +* `pack_bits(bits)`: Given the `bits` of bitlist or bitvector, get `bitfield_bytes` by packing them in bytes and aligning to the start. The length-delimiting bit for bitlists is excluded. And then pack `bitfield_bytes` into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` * `merkleize(chunks, limit=None)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, merkleize the chunks, and return the root: * The merkleization depends on the effective input, which can be padded/limited: @@ -228,9 +228,9 @@ We first define helper functions: We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects. -* `merkleize(bitfield_bytes(value), limit=chunk_count(type))` if `value` is a bitvector. +* `merkleize(pack_bits(value), limit=chunk_count(type))` if `value` is a bitvector. * `mix_in_length(merkleize(pack(value), limit=chunk_count(type)), len(value))` if `value` is a list of basic objects. -* `mix_in_length(merkleize(bitfield_bytes(value), limit=chunk_count(type)), len(value))` if `value` is a bitlist. +* `mix_in_length(merkleize(pack_bits(value), limit=chunk_count(type)), len(value))` if `value` is a bitlist. * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container. * `mix_in_length(merkleize([hash_tree_root(element) for element in value], limit=chunk_count(type)), len(value))` if `value` is a list of composite objects. * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type. diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index a8839f70de..d33c3a2128 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -0.11.2 \ No newline at end of file +0.12.0 \ No newline at end of file diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 4c5768a294..c43c1521bc 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -24,12 +24,12 @@ def apply_constants_config(spec_globals: Dict[str, Any], warn_if_unknown: bool = # Load presets from a file, and then prepares the global config setting. This does not apply the config. # To apply the config, reload the spec module (it will re-initialize with the config taken from here). -def prepare_config(configs_path, config_name): +def prepare_config(configs_path: str, config_name: str) -> None: global config config = load_config_file(configs_path, config_name) -def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: +def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: """ Loads the given preset :param presets_name: The name of the presets. (lowercase snake_case) @@ -38,7 +38,7 @@ def load_config_file(configs_dir, presets_name) -> Dict[str, Any]: path = Path(join(configs_dir, presets_name + '.yaml')) yaml = YAML(typ='base') loaded = yaml.load(path) - out = dict() + out: Dict[str, Any] = dict() for k, v in loaded.items(): if isinstance(v, list): # Clean up integer values. YAML parser renders lists of ints as list of str diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 1a182fd312..20214908e2 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -9,6 +9,8 @@ from random import Random from typing import Any, Callable, NewType, Sequence, TypedDict, Protocol +from lru import LRU + from importlib import reload @@ -48,28 +50,46 @@ class SpecForks(TypedDict, total=False): PHASE1: SpecPhase1 +def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int], + spec: Spec, phases: SpecForks): + + p0 = phases[PHASE0] + balances = balances_fn(p0) + activation_threshold = threshold_fn(p0) + + state = create_genesis_state(spec=p0, validator_balances=balances, + activation_threshold=activation_threshold) + if spec.fork == PHASE1: + # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. + # Decide based on performance/consistency results later. + state = phases[PHASE1].upgrade_to_phase1(state) + # Shard state slot must lag behind BeaconState slot by at least 1 + # Will handle this more elegantly with fork mechanics + spec.process_slots(state, state.slot + 1) + + return state + + +_custom_state_cache_dict = LRU(size=10) + + def with_custom_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int]): def deco(fn): + def entry(*args, spec: Spec, phases: SpecForks, **kw): - try: - p0 = phases[PHASE0] - balances = balances_fn(p0) - activation_threshold = threshold_fn(p0) - - state = create_genesis_state(spec=p0, validator_balances=balances, - activation_threshold=activation_threshold) - if spec.fork == PHASE1: - # TODO: instead of upgrading a test phase0 genesis state we can also write a phase1 state helper. - # Decide based on performance/consistency results later. - state = phases[PHASE1].upgrade_to_phase1(state) - # Shard state slot must lag behind BeaconState slot by at least 1 - # Will handle this more elegantly with fork mechanics - spec.process_slots(state, state.slot + 1) - - kw['state'] = state - except KeyError: - raise TypeError('Spec decorator must come within state decorator to inject spec into state.') + # make a key for the state + # genesis fork version separates configs during test-generation runtime. + key = (spec.fork, spec.GENESIS_FORK_VERSION, spec.__file__, balances_fn, threshold_fn) + global _custom_state_cache_dict + if key not in _custom_state_cache_dict: + state = _prepare_state(balances_fn, threshold_fn, spec, phases) + _custom_state_cache_dict[key] = state.get_backing() + + # Take an entry out of the LRU. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_custom_state_cache_dict[key]) + kw['state'] = state return fn(*args, spec=spec, phases=phases, **kw) return entry return deco diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 360c18ccd1..b2d33d0aa7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -1,7 +1,7 @@ from eth2spec.test.context import PHASE0, with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation -from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch +from eth2spec.test.helpers.state import transition_to, state_transition_and_sign_block, next_epoch, next_slot def run_on_attestation(spec, state, store, attestation, valid=True): @@ -116,6 +116,44 @@ def test_on_attestation_mismatched_target_and_slot(spec, state): run_on_attestation(spec, state, store, attestation, False) +@with_all_phases +@spec_state_test +def test_on_attestation_inconsistent_target_and_head(spec, state): + store = spec.get_forkchoice_store(state) + spec.on_tick(store, store.time + 2 * spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH) + + # Create chain 1 as empty chain between genesis and start of 1st epoch + target_state_1 = state.copy() + next_epoch(spec, target_state_1) + + # Create chain 2 with different block in chain from chain 1 from chain 1 from chain 1 from chain 1 + target_state_2 = state.copy() + diff_block = build_empty_block_for_next_slot(spec, target_state_2) + signed_diff_block = state_transition_and_sign_block(spec, target_state_2, diff_block) + spec.on_block(store, signed_diff_block) + next_epoch(spec, target_state_2) + next_slot(spec, target_state_2) + + # Create and store block new head block on target state 1 + head_block = build_empty_block_for_next_slot(spec, target_state_1) + signed_head_block = state_transition_and_sign_block(spec, target_state_1, head_block) + spec.on_block(store, signed_head_block) + + # Attest to head of chain 1 + attestation = get_valid_attestation(spec, target_state_1, slot=head_block.slot, signed=False) + epoch = spec.compute_epoch_at_slot(attestation.data.slot) + + # Set attestation target to be from chain 2 + attestation.data.target = spec.Checkpoint(epoch=epoch, root=spec.get_block_root(target_state_2, epoch)) + sign_attestation(spec, state, attestation) + + assert attestation.data.target.epoch == spec.GENESIS_EPOCH + 1 + assert spec.compute_epoch_at_slot(attestation.data.slot) == spec.GENESIS_EPOCH + 1 + assert spec.get_block_root(target_state_1, epoch) != attestation.data.target.root + + run_on_attestation(spec, state, store, attestation, False) + + @with_all_phases @spec_state_test def test_on_attestation_target_not_in_store(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py index 4438dff920..016326b30c 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -184,7 +184,7 @@ def test_on_block_finalized_skip_slots_not_in_skip_chain(spec, state): def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state): # Initialization store = spec.get_forkchoice_store(state) - time = 100 + time = 0 spec.on_tick(store, time) next_epoch(spec, state) @@ -215,7 +215,7 @@ def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state): def test_on_block_outside_safe_slots_and_multiple_better_justified(spec, state): # Initialization store = spec.get_forkchoice_store(state) - time = 100 + time = 0 spec.on_tick(store, time) next_epoch(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index e93a63c283..79f752411a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -6,6 +6,7 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist +from lru import LRU def run_attestation_processing(spec, state, attestation, valid=True): @@ -148,10 +149,12 @@ def get_valid_attestation(spec, state, slot=None, index=None, + filter_participant_set=None, shard_transition=None, - empty=False, signed=False, on_time=True): + # If filter_participant_set filters everything, the attestation has 0 participants, and cannot be signed. + # Thus strictly speaking invalid when no participant is added later. if slot is None: slot = state.slot if index is None: @@ -173,10 +176,8 @@ def get_valid_attestation(spec, aggregation_bits=aggregation_bits, data=attestation_data, ) - if not empty: - fill_aggregate_attestation(spec, state, attestation) - if signed: - sign_attestation(spec, state, attestation) + # fill the attestation with (optionally filtered) participants, and optionally sign it + fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set) if spec.fork == PHASE1 and on_time: attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) @@ -269,16 +270,25 @@ def get_attestation_signature(spec, state, attestation_data, privkey): return bls.Sign(privkey, signing_root) -def fill_aggregate_attestation(spec, state, attestation, signed=False): +def fill_aggregate_attestation(spec, state, attestation, signed=False, filter_participant_set=None): + """ + `signed`: Signing is optional. + `filter_participant_set`: Optional, filters the full committee indices set (default) to a subset that participates + """ beacon_committee = spec.get_beacon_committee( state, attestation.data.slot, attestation.data.index, ) + # By default, have everyone participate + participants = set(beacon_committee) + # But optionally filter the participants to a smaller amount + if filter_participant_set is not None: + participants = filter_participant_set(participants) for i in range(len(beacon_committee)): - attestation.aggregation_bits[i] = True + attestation.aggregation_bits[i] = beacon_committee[i] in participants - if signed: + if signed and len(participants) > 0: sign_attestation(spec, state, attestation) @@ -324,10 +334,12 @@ def next_epoch_with_attestations(spec, return state, signed_blocks, post_state -def prepare_state_with_full_attestations(spec, state, empty=False): +def prepare_state_with_attestations(spec, state, participation_fn=None): """ - Fill ``state`` with maximally full attestations. - Move to the start of the next epoch to ensure full epoch worth. + Prepare state with attestations according to the ``participation_fn``. + If no ``participation_fn``, default to "full" -- max committee participation at each slot. + + participation_fn: (slot, committee_index, committee_indices_set) -> participants_indices_set """ # Go to start of next epoch to ensure can have full participation next_epoch(spec, state) @@ -340,8 +352,15 @@ def prepare_state_with_full_attestations(spec, state, empty=False): # create an attestation for each index in each slot in epoch if state.slot < next_epoch_start_slot: for committee_index in range(spec.get_committee_count_at_slot(state, state.slot)): - attestation = get_valid_attestation(spec, state, index=committee_index, empty=empty, signed=True) - attestations.append(attestation) + def temp_participants_filter(comm): + if participation_fn is None: + return comm + else: + return participation_fn(state.slot, committee_index, comm) + attestation = get_valid_attestation(spec, state, index=committee_index, + filter_participant_set=temp_participants_filter, signed=True) + if any(attestation.aggregation_bits): # Only if there is at least 1 participant. + attestations.append(attestation) # fill each created slot in state after inclusion delay if state.slot >= start_slot + spec.MIN_ATTESTATION_INCLUSION_DELAY: inclusion_slot = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY @@ -355,6 +374,27 @@ def prepare_state_with_full_attestations(spec, state, empty=False): return attestations +_prep_state_cache_dict = LRU(size=10) + + +def cached_prepare_state_with_attestations(spec, state): + """ + Cached version of prepare_state_with_attestations, + but does not return anything, and does not support a participation fn argument + """ + # If the pre-state is not already known in the LRU, then take it, + # prepare it with attestations, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. + key = (spec.fork, state.hash_tree_root()) + global _prep_state_cache_dict + if key not in _prep_state_cache_dict: + prepare_state_with_attestations(spec, state) + _prep_state_cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. + + # Put the LRU cache result into the state view, as if we transitioned the original view + state.set_backing(_prep_state_cache_dict[key]) + + def fill_block_shard_transitions_by_attestations(spec, state, block): block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS for attestation in block.body.attestations: diff --git a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py index 975f34c209..e743ca8ff6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -1,5 +1,5 @@ from eth2spec.test.context import PHASE1 -from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation +from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation, sign_indexed_attestation def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): @@ -17,6 +17,26 @@ def get_valid_attester_slashing(spec, state, signed_1=False, signed_2=False): ) +def get_valid_attester_slashing_by_indices(spec, state, indices_1, indices_2=None, signed_1=False, signed_2=False): + if indices_2 is None: + indices_2 = indices_1 + + assert indices_1 == sorted(indices_1) + assert indices_2 == sorted(indices_2) + + attester_slashing = get_valid_attester_slashing(spec, state) + + attester_slashing.attestation_1.attesting_indices = indices_1 + attester_slashing.attestation_2.attesting_indices = indices_2 + + if signed_1: + sign_indexed_attestation(spec, state, attester_slashing.attestation_1) + if signed_2: + sign_indexed_attestation(spec, state, attester_slashing.attestation_2) + + return attester_slashing + + def get_indexed_attestation_participants(spec, indexed_att): """ Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. diff --git a/tests/core/pyspec/eth2spec/test/helpers/deposits.py b/tests/core/pyspec/eth2spec/test/helpers/deposits.py index a16f7a7bf8..6a2e30497e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/deposits.py +++ b/tests/core/pyspec/eth2spec/test/helpers/deposits.py @@ -5,6 +5,17 @@ from eth2spec.utils.ssz.ssz_typing import List +def mock_deposit(spec, state, index): + """ + Mock validator at ``index`` as having just made a deposit + """ + assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + state.validators[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE + assert not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) + + def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, signed=False): deposit_data = spec.DepositData( pubkey=pubkey, diff --git a/tests/core/pyspec/eth2spec/test/helpers/keys.py b/tests/core/pyspec/eth2spec/test/helpers/keys.py index 7f7820d3a0..d813870e05 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/keys.py +++ b/tests/core/pyspec/eth2spec/test/helpers/keys.py @@ -2,5 +2,5 @@ from eth2spec.phase0 import spec privkeys = [i + 1 for i in range(spec.SLOTS_PER_EPOCH * 256)] -pubkeys = [bls.PrivToPub(privkey) for privkey in privkeys] +pubkeys = [bls.SkToPk(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index eaeb9252e4..d62fee6ce9 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,7 +1,8 @@ from random import Random from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations +from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations +from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch from eth2spec.utils.ssz.ssz_typing import Container, uint64, List @@ -24,17 +25,50 @@ def has_enough_for_reward(spec, state, index): ) -def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn): +def run_deltas(spec, state): """ - Run ``component_delta_fn``, yielding: + Run all deltas functions yielding: - pre-state ('pre') - - deltas ('deltas') + - source deltas ('source_deltas') + - target deltas ('target_deltas') + - head deltas ('head_deltas') + - inclusion delay deltas ('inclusion_delay_deltas') + - inactivity penalty deltas ('inactivity_penalty_deltas') """ yield 'pre', state + yield from run_attestation_component_deltas( + spec, + state, + spec.get_source_deltas, + spec.get_matching_source_attestations, + 'source_deltas', + ) + yield from run_attestation_component_deltas( + spec, + state, + spec.get_target_deltas, + spec.get_matching_target_attestations, + 'target_deltas', + ) + yield from run_attestation_component_deltas( + spec, + state, + spec.get_head_deltas, + spec.get_matching_head_attestations, + 'head_deltas', + ) + yield from run_get_inclusion_delay_deltas(spec, state) + yield from run_get_inactivity_penalty_deltas(spec, state) + +def run_attestation_component_deltas(spec, state, component_delta_fn, matching_att_fn, deltas_name): + """ + Run ``component_delta_fn``, yielding: + - deltas ('{``deltas_name``}') + """ rewards, penalties = component_delta_fn(state) - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) + yield deltas_name, Deltas(rewards=rewards, penalties=penalties) matching_attestations = matching_att_fn(state, spec.get_previous_epoch(state)) matching_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) @@ -61,6 +95,95 @@ def run_attestation_component_deltas(spec, state, component_delta_fn, matching_a assert penalties[index] == 0 +def run_get_inclusion_delay_deltas(spec, state): + """ + Run ``get_inclusion_delay_deltas``, yielding: + - inclusion delay deltas ('inclusion_delay_deltas') + """ + rewards, penalties = spec.get_inclusion_delay_deltas(state) + + yield 'inclusion_delay_deltas', Deltas(rewards=rewards, penalties=penalties) + + eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) + attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) + + rewarded_indices = set() + rewarded_proposer_indices = set() + # Ensure attesters with enough balance are rewarded for attestations + # Track those that are rewarded and track proposers that should be rewarded + for index in range(len(state.validators)): + if index in attesting_indices and has_enough_for_reward(spec, state, index): + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Track proposer of earliest included attestation for the validator defined by index + earliest_attestation = min([ + a for a in eligible_attestations + if index in spec.get_attesting_indices(state, a.data, a.aggregation_bits) + ], key=lambda a: a.inclusion_delay) + rewarded_proposer_indices.add(earliest_attestation.proposer_index) + + # Ensure all expected proposers have been rewarded + # Track rewarde indices + proposing_indices = [a.proposer_index for a in eligible_attestations] + for index in proposing_indices: + if index in rewarded_proposer_indices: + assert rewards[index] > 0 + rewarded_indices.add(index) + + # Ensure all expected non-rewarded indices received no reward + for index in range(len(state.validators)): + assert penalties[index] == 0 + if index not in rewarded_indices: + assert rewards[index] == 0 + + +def run_get_inactivity_penalty_deltas(spec, state): + """ + Run ``get_inactivity_penalty_deltas``, yielding: + - inactivity penalty deltas ('inactivity_penalty_deltas') + """ + rewards, penalties = spec.get_inactivity_penalty_deltas(state) + + yield 'inactivity_penalty_deltas', Deltas(rewards=rewards, penalties=penalties) + + matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) + matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) + + finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch + eligible_indices = spec.get_eligible_validator_indices(state) + for index in range(len(state.validators)): + assert rewards[index] == 0 + if index not in eligible_indices: + assert penalties[index] == 0 + continue + + if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: + base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) + if not has_enough_for_reward(spec, state, index): + assert penalties[index] == 0 + elif index in matching_attesting_indices: + assert penalties[index] == base_penalty + else: + assert penalties[index] > base_penalty + else: + assert penalties[index] == 0 + + +def set_some_new_deposits(spec, state, rng): + num_validators = len(state.validators) + # Set ~1/10 to just recently deposited + for index in range(num_validators): + # If not already active, skip + if not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)): + continue + if rng.randrange(num_validators) < num_validators // 10: + mock_deposit(spec, state, index) + # Set ~half of selected to eligible for activation + if rng.choice([True, False]): + state.validators[index].activation_eligibility_epoch = spec.get_current_epoch(state) + + def exit_random_validators(spec, state, rng): if spec.get_current_epoch(state) < 5: # Move epochs forward to allow for some validators already exited/withdrawable @@ -69,10 +192,11 @@ def exit_random_validators(spec, state, rng): current_epoch = spec.get_current_epoch(state) # Exit ~1/2 of validators - for validator in state.validators: + for index in spec.get_active_validator_indices(state, current_epoch): if rng.choice([True, False]): continue + validator = state.validators[index] validator.exit_epoch = rng.choice([current_epoch - 1, current_epoch - 2, current_epoch - 3]) # ~1/2 are withdrawable if rng.choice([True, False]): @@ -83,84 +207,92 @@ def exit_random_validators(spec, state, rng): def slash_random_validators(spec, state, rng): # Slash ~1/2 of validators - for validator in state.validators: - validator.slashed = rng.choice([True, False]) + for index in range(len(state.validators)): + # slash at least one validator + if index == 0 or rng.choice([True, False]): + spec.slash_validator(state, index) -def run_test_empty(spec, state, runner): +def run_test_empty(spec, state): # Do not add any attestations to state - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_all_correct(spec, state, runner): - prepare_state_with_full_attestations(spec, state) +def run_test_full_all_correct(spec, state): + cached_prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_but_partial_participation(spec, state, runner, rng=Random(5522)): - prepare_state_with_full_attestations(spec, state) +def run_test_full_but_partial_participation(spec, state, rng=Random(5522)): + cached_prepare_state_with_attestations(spec, state) for a in state.previous_epoch_attestations: a.aggregation_bits = [rng.choice([True, False]) for _ in a.aggregation_bits] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_partial(spec, state, fraction_filled, runner): - prepare_state_with_full_attestations(spec, state) +def run_test_partial(spec, state, fraction_filled): + cached_prepare_state_with_attestations(spec, state) # Remove portion of attestations num_attestations = int(len(state.previous_epoch_attestations) * fraction_filled) state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_half_full(spec, state, runner): - yield from run_test_partial(spec, state, 0.5, runner) +def run_test_half_full(spec, state): + yield from run_test_partial(spec, state, 0.5) -def run_test_one_attestation_one_correct(spec, state, runner): - prepare_state_with_full_attestations(spec, state) +def run_test_one_attestation_one_correct(spec, state): + cached_prepare_state_with_attestations(spec, state) # Remove all attestations except for the first one state.previous_epoch_attestations = state.previous_epoch_attestations[:1] - yield from runner(spec, state) + yield from run_deltas(spec, state) + + +def run_test_with_not_yet_activated_validators(spec, state, rng=Random(5555)): + set_some_new_deposits(spec, state, rng) + cached_prepare_state_with_attestations(spec, state) + yield from run_deltas(spec, state) -def run_test_with_exited_validators(spec, state, runner, rng=Random(1337)): + +def run_test_with_exited_validators(spec, state, rng=Random(1337)): exit_random_validators(spec, state, rng) - prepare_state_with_full_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_with_slashed_validators(spec, state, runner, rng=Random(3322)): +def run_test_with_slashed_validators(spec, state, rng=Random(3322)): exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) - prepare_state_with_full_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_some_very_low_effective_balances_that_attested(spec, state, runner): - state.balances - prepare_state_with_full_attestations(spec, state) +def run_test_some_very_low_effective_balances_that_attested(spec, state): + cached_prepare_state_with_attestations(spec, state) # Set some balances to be very low (including 0) assert len(state.validators) >= 5 for i, index in enumerate(range(5)): state.validators[index].effective_balance = i - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state, runner): - prepare_state_with_full_attestations(spec, state) +def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state): + cached_prepare_state_with_attestations(spec, state) # Remove attestation attestation = state.previous_epoch_attestations[0] @@ -170,11 +302,11 @@ def run_test_some_very_low_effective_balances_that_did_not_attest(spec, state, r for i, index in enumerate(indices): state.validators[index].effective_balance = i - yield from runner(spec, state) + yield from run_deltas(spec, state) -def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect, runner): - prepare_state_with_full_attestations(spec, state) +def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, fraction_incorrect): + cached_prepare_state_with_attestations(spec, state) # Make fraction_incorrect of pending attestations have bad target/head as specified num_incorrect = int(fraction_incorrect * len(state.previous_epoch_attestations)) @@ -184,14 +316,97 @@ def run_test_full_fraction_incorrect(spec, state, correct_target, correct_head, if not correct_head: pending_attestation.data.beacon_block_root = b'\x66' * 32 - yield from runner(spec, state) + yield from run_deltas(spec, state) + + +def run_test_full_delay_one_slot(spec, state): + cached_prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += 1 + + yield from run_deltas(spec, state) + + +def run_test_full_delay_max_slots(spec, state): + cached_prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay += spec.SLOTS_PER_EPOCH + + yield from run_deltas(spec, state) + + +def run_test_full_mixed_delay(spec, state, rng=Random(1234)): + cached_prepare_state_with_attestations(spec, state) + for a in state.previous_epoch_attestations: + a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) + + yield from run_deltas(spec, state) + + +def run_test_proposer_not_in_attestations(spec, state): + cached_prepare_state_with_attestations(spec, state) + + # Get an attestation where the proposer is not in the committee + non_proposer_attestations = [] + for a in state.previous_epoch_attestations: + if a.proposer_index not in spec.get_unslashed_attesting_indices(state, [a]): + non_proposer_attestations.append(a) + + assert any(non_proposer_attestations) + state.previous_epoch_attestations = non_proposer_attestations + + yield from run_deltas(spec, state) + + +def run_test_duplicate_attestations_at_later_slots(spec, state): + cached_prepare_state_with_attestations(spec, state) + + # Remove 2/3 of attestations to make it more interesting + num_attestations = int(len(state.previous_epoch_attestations) * 0.33) + state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] + + # Get map of the proposer at each slot to make valid-looking duplicate attestations + per_slot_proposers = { + (a.data.slot + a.inclusion_delay): a.proposer_index + for a in state.previous_epoch_attestations + } + max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) + later_attestations = [] + for a in state.previous_epoch_attestations: + # Only have proposers for previous epoch so do not create later + # duplicate if slot exceeds the max slot in previous_epoch_attestations + if a.data.slot + a.inclusion_delay >= max_slot: + continue + later_a = a.copy() + later_a.inclusion_delay += 1 + later_a.proposer_index = per_slot_proposers[later_a.data.slot + later_a.inclusion_delay] + later_attestations.append(later_a) + + assert any(later_attestations) + + state.previous_epoch_attestations = sorted( + state.previous_epoch_attestations + later_attestations, + key=lambda a: a.data.slot + a.inclusion_delay + ) + + yield from run_deltas(spec, state) + + +def run_test_all_balances_too_low_for_reward(spec, state): + cached_prepare_state_with_attestations(spec, state) + + for index in range(len(state.validators)): + state.validators[index].effective_balance = 10 + + yield from run_deltas(spec, state) -def run_test_full_random(spec, state, runner, rng=Random(8020)): +def run_test_full_random(spec, state, rng=Random(8020)): + set_some_new_deposits(spec, state, rng) exit_random_validators(spec, state, rng) slash_random_validators(spec, state, rng) - prepare_state_with_full_attestations(spec, state) + cached_prepare_state_with_attestations(spec, state) for pending_attestation in state.previous_epoch_attestations: # ~1/3 have bad target @@ -205,4 +420,4 @@ def run_test_full_random(spec, state, runner, rng=Random(8020)): # Random inclusion delay pending_attestation.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) - yield from runner(spec, state) + yield from run_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 8752d37479..5de8913d6e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -13,7 +13,6 @@ sign_attestation, ) from eth2spec.test.helpers.state import ( - next_slot, next_slots, next_epoch_via_block, transition_to_slot_via_block, @@ -61,6 +60,29 @@ def test_invalid_attestation_signature(spec, state): yield from run_attestation_processing(spec, state, attestation, False) +@with_all_phases +@spec_state_test +@always_bls +def test_empty_participants_zeroes_sig(spec, state): + attestation = get_valid_attestation(spec, state, filter_participant_set=lambda comm: []) # 0 participants + attestation.signature = spec.BLSSignature(b'\x00' * 96) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation, False) + + +@with_all_phases +@spec_state_test +@always_bls +def test_empty_participants_seemingly_valid_sig(spec, state): + attestation = get_valid_attestation(spec, state, filter_participant_set=lambda comm: []) # 0 participants + # Special BLS value, valid for zero pubkeys on some (but not all) BLS implementations. + attestation.signature = spec.BLSSignature(b'\xc0' + b'\x00' * 95) + next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) + + yield from run_attestation_processing(spec, state, attestation, False) + + @with_all_phases @spec_state_test def test_before_inclusion_delay(spec, state): @@ -256,19 +278,6 @@ def test_bad_source_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases -@spec_state_test -def test_empty_aggregation_bits(spec, state): - next_slot(spec, state) - attestation = get_valid_attestation(spec, state, empty=True) - next_slots(spec, state, spec.MIN_ATTESTATION_INCLUSION_DELAY) - - assert attestation.aggregation_bits == Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]( - *([0b0] * len(attestation.aggregation_bits))) - - yield from run_attestation_processing(spec, state, attestation) - - @with_all_phases @spec_state_test def test_too_many_aggregation_bits(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 19915750f6..9464f80aaf 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -34,8 +34,8 @@ def run_voluntary_exit_processing(spec, state, signed_voluntary_exit, valid=True @with_all_phases @spec_state_test def test_success(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -53,8 +53,8 @@ def test_success(spec, state): @spec_state_test @always_bls def test_invalid_signature(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -71,8 +71,8 @@ def test_invalid_signature(spec, state): @with_all_phases @spec_state_test def test_success_exit_queue(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) @@ -115,8 +115,8 @@ def test_success_exit_queue(spec, state): @with_all_phases @spec_state_test def test_default_exit_epoch_subsequent_exit(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -137,8 +137,8 @@ def test_default_exit_epoch_subsequent_exit(spec, state): @with_all_phases @spec_state_test def test_validator_exit_in_future(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -156,8 +156,8 @@ def test_validator_exit_in_future(spec, state): @with_all_phases @spec_state_test def test_validator_invalid_validator_index(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -190,8 +190,8 @@ def test_validator_not_active(spec, state): @with_all_phases @spec_state_test def test_validator_already_exited(spec, state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow validator able to exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -218,7 +218,7 @@ def test_validator_not_active_long_enough(spec, state): assert ( current_epoch - state.validators[validator_index].activation_epoch < - spec.PERSISTENT_COMMITTEE_PERIOD + spec.SHARD_COMMITTEE_PERIOD ) yield from run_voluntary_exit_processing(spec, state, signed_voluntary_exit, False) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index a5f4d92279..b6597b1cf6 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,3 +1,4 @@ +from eth2spec.test.helpers.deposits import mock_deposit from eth2spec.test.helpers.state import next_epoch, next_slots from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with @@ -7,14 +8,6 @@ def run_process_registry_updates(spec, state): yield from run_epoch_processing_with(spec, state, 'process_registry_updates') -def mock_deposit(spec, state, index): - assert spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) - state.validators[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - state.validators[index].activation_epoch = spec.FAR_FUTURE_EPOCH - state.validators[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE - assert not spec.is_active_validator(state.validators[index], spec.get_current_epoch(state)) - - @with_all_phases @spec_state_test def test_add_to_activation_queue(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index f1d86c373a..eff2864484 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -12,10 +12,11 @@ from eth2spec.test.helpers.attestations import ( add_attestations_to_state, get_valid_attestation, - prepare_state_with_full_attestations, + prepare_state_with_attestations, ) from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from random import Random def run_process_rewards_and_penalties(spec, state): @@ -64,7 +65,7 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): @with_all_phases @spec_state_test def test_full_attestations(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) pre_state = state.copy() @@ -82,7 +83,7 @@ def test_full_attestations(spec, state): @with_all_phases @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) for i, attestation in enumerate(state.previous_epoch_attestations): if i % 3 == 0: # Mess up some head votes @@ -107,7 +108,7 @@ def test_full_attestations_random_incorrect_fields(spec, state): @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.MAX_EFFECTIVE_BALANCE // 2) @single_phase def test_full_attestations_misc_balances(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) pre_state = state.copy() @@ -139,7 +140,7 @@ def test_full_attestations_misc_balances(spec, state): @with_custom_state(balances_fn=low_single_balance, threshold_fn=zero_activation_threshold) @single_phase def test_full_attestations_one_validaor_one_gwei(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) yield from run_process_rewards_and_penalties(spec, state) @@ -163,20 +164,55 @@ def test_no_attestations_all_penalties(spec, state): assert state.balances[index] < pre_state.balances[index] -@with_all_phases -@spec_state_test -def test_empty_attestations(spec, state): - attestations = prepare_state_with_full_attestations(spec, state, empty=True) +def run_with_participation(spec, state, participation_fn): + participated = set() + + def participation_tracker(slot, comm_index, comm): + att_participants = participation_fn(slot, comm_index, comm) + participated.update(att_participants) + return att_participants + + attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) pre_state = state.copy() yield from run_process_rewards_and_penalties(spec, state) attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) == 0 + assert len(attesting_indices) == len(participated) for index in range(len(pre_state.validators)): - assert state.balances[index] < pre_state.balances[index] + if index in participated: + assert state.balances[index] > pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] + + +@with_all_phases +@spec_state_test +def test_almost_empty_attestations(spec, state): + rng = Random(1234) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) + + +@with_all_phases +@spec_state_test +def test_random_fill_attestations(spec, state): + rng = Random(4567) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) + + +@with_all_phases +@spec_state_test +def test_almost_full_attestations(spec, state): + rng = Random(8901) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) + + +@with_all_phases +@spec_state_test +def test_full_attestation_participation(spec, state): + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) @with_all_phases @@ -221,7 +257,7 @@ def test_duplicate_attestation(spec, state): @spec_state_test # Case when some eligible attestations are slashed. Modifies attesting_balance and consequently rewards/penalties. def test_attestations_some_slashed(spec, state): - attestations = prepare_state_with_full_attestations(spec, state) + attestations = prepare_state_with_attestations(spec, state) attesting_indices_before_slashings = list(spec.get_unslashed_attesting_indices(state, attestations)) # Slash maximum amount of validators allowed per epoch. diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py similarity index 63% rename from tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py rename to tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py index 54f8f3b5de..92277fdd7e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_source_deltas.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_basic.py @@ -1,83 +1,71 @@ from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas import eth2spec.test.helpers.rewards as rewards_helpers -def run_get_source_deltas(spec, state): - """ - Run ``get_source_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_source_deltas, - spec.get_matching_source_attestations, - ) - - @with_all_phases @spec_state_test def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_empty(spec, state) @with_all_phases @spec_state_test def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_all_correct(spec, state) @with_all_phases @spec_state_test def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_half_full(spec, state) + + +@with_all_phases +@spec_state_test +def test_quarter_full(spec, state): + yield from rewards_helpers.run_test_partial(spec, state, 0.25) @with_all_phases @spec_state_test def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) @with_all_phases @spec_state_test def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) + + +@with_all_phases +@spec_state_test +def test_with_not_yet_activated_validators(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) @with_all_phases @spec_state_test def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_exited_validators(spec, state) @with_all_phases @spec_state_test def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_source_deltas) + yield from rewards_helpers.run_test_with_slashed_validators(spec, state) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_source_deltas - ) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) @with_all_phases @spec_state_test def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_source_deltas, - ) + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) # @@ -95,7 +83,6 @@ def test_full_half_correct_target_incorrect_head(spec, state): correct_target=True, correct_head=False, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @@ -107,7 +94,6 @@ def test_full_correct_target_incorrect_head(spec, state): correct_target=True, correct_head=False, fraction_incorrect=1.0, - runner=run_get_source_deltas ) @@ -119,7 +105,6 @@ def test_full_half_incorrect_target_incorrect_head(spec, state): correct_target=False, correct_head=False, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @@ -131,11 +116,40 @@ def test_full_half_incorrect_target_correct_head(spec, state): correct_target=False, correct_head=True, fraction_incorrect=0.5, - runner=run_get_source_deltas ) @with_all_phases @spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_source_deltas) +def test_full_delay_one_slot(spec, state): + yield from rewards_helpers.run_test_full_delay_one_slot(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_delay_max_slots(spec, state): + yield from rewards_helpers.run_test_full_delay_max_slots(spec, state) + + +@with_all_phases +@spec_state_test +def test_full_mixed_delay(spec, state): + yield from rewards_helpers.run_test_full_mixed_delay(spec, state) + + +@with_all_phases +@spec_state_test +def test_proposer_not_in_attestations(spec, state): + yield from rewards_helpers.run_test_proposer_not_in_attestations(spec, state) + + +@with_all_phases +@spec_state_test +def test_duplicate_attestations_at_later_slots(spec, state): + yield from rewards_helpers.run_test_duplicate_attestations_at_later_slots(spec, state) + + +@with_all_phases +@spec_state_test +def test_all_balances_too_low_for_reward(spec, state): + yield from rewards_helpers.run_test_all_balances_too_low_for_reward(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py deleted file mode 100644 index 2e4b9dbbcb..0000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_head_deltas.py +++ /dev/null @@ -1,130 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_head_deltas(spec, state): - """ - Run ``get_head_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_head_deltas, - spec.get_matching_head_attestations, - ) - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state, run_get_head_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_head_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=1.0, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=True, - fraction_incorrect=0.5, - runner=run_get_head_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_head_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py deleted file mode 100644 index 4940cdc63d..0000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inactivity_penalty_deltas.py +++ /dev/null @@ -1,210 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import has_enough_for_reward -from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.helpers.rewards import Deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_inactivity_penalty_deltas(spec, state): - """ - Run ``get_inactivity_penalty_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield 'pre', state - - rewards, penalties = spec.get_inactivity_penalty_deltas(state) - - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) - - matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) - matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) - - finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch - eligible_indices = spec.get_eligible_validator_indices(state) - for index in range(len(state.validators)): - assert rewards[index] == 0 - if index not in eligible_indices: - assert penalties[index] == 0 - continue - - if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: - base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) - if not has_enough_for_reward(spec, state, index): - assert penalties[index] == 0 - elif index in matching_attesting_indices: - assert penalties[index] == base_penalty - else: - assert penalties[index] > base_penalty - else: - assert penalties[index] == 0 - - -def transition_state_to_leak(spec, state, epochs=None): - if epochs is None: - epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - - for _ in range(epochs): - next_epoch(spec, state) - - -@with_all_phases -@spec_state_test -def test_empty_no_leak(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_empty_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_empty(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_no_leak(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full_no_leak(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full_no_leak(spec, state): - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation_no_leak(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators_no_leak(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested_no_leak(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest_no_leak(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_inactivity_penalty_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_random_no_leak(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_leak(spec, state): - transition_state_to_leak(spec, state) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_five_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=5) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) - - -@with_all_phases -@spec_state_test -def test_full_random_ten_epoch_leak(spec, state): - transition_state_to_leak(spec, state, epochs=10) - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inactivity_penalty_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py deleted file mode 100644 index 526d135ede..0000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_inclusion_delay_deltas.py +++ /dev/null @@ -1,207 +0,0 @@ -from random import Random - -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.attestations import prepare_state_with_full_attestations -from eth2spec.test.helpers.rewards import Deltas, has_enough_for_reward -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_inclusion_delay_deltas(spec, state): - """ - Run ``get_inclusion_delay_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield 'pre', state - - rewards, penalties = spec.get_inclusion_delay_deltas(state) - - yield 'deltas', Deltas(rewards=rewards, penalties=penalties) - - eligible_attestations = spec.get_matching_source_attestations(state, spec.get_previous_epoch(state)) - attesting_indices = spec.get_unslashed_attesting_indices(state, eligible_attestations) - - rewarded_indices = set() - rewarded_proposer_indices = set() - # Ensure attesters with enough balance are rewarded for attestations - # Track those that are rewarded and track proposers that should be rewarded - for index in range(len(state.validators)): - if index in attesting_indices and has_enough_for_reward(spec, state, index): - assert rewards[index] > 0 - rewarded_indices.add(index) - - # Track proposer of earliest included attestation for the validator defined by index - earliest_attestation = min([ - a for a in eligible_attestations - if index in spec.get_attesting_indices(state, a.data, a.aggregation_bits) - ], key=lambda a: a.inclusion_delay) - rewarded_proposer_indices.add(earliest_attestation.proposer_index) - - # Ensure all expected proposers have been rewarded - # Track rewarde indices - proposing_indices = [a.proposer_index for a in eligible_attestations] - for index in proposing_indices: - if index in rewarded_proposer_indices: - assert rewards[index] > 0 - rewarded_indices.add(index) - - # Ensure all expected non-rewarded indices received no reward - for index in range(len(state.validators)): - assert penalties[index] == 0 - if index not in rewarded_indices: - assert rewards[index] == 0 - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_quarter_full(spec, state): - yield from rewards_helpers.run_test_partial(spec, state, 0.25, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_exited_validators(spec, state): - yield from rewards_helpers.run_test_with_exited_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_inclusion_delay_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_inclusion_delay_deltas) - - -@with_all_phases -@spec_state_test -def test_full_delay_one_slot(spec, state): - prepare_state_with_full_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay += 1 - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_full_delay_max_slots(spec, state): - prepare_state_with_full_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay += spec.SLOTS_PER_EPOCH - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_full_mixed_delay(spec, state): - rng = Random(1234) - - prepare_state_with_full_attestations(spec, state) - for a in state.previous_epoch_attestations: - a.inclusion_delay = rng.randint(1, spec.SLOTS_PER_EPOCH) - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_proposer_not_in_attestations(spec, state): - prepare_state_with_full_attestations(spec, state) - - # Get an attestation where the proposer is not in the committee - non_proposer_attestations = [] - for a in state.previous_epoch_attestations: - if a.proposer_index not in spec.get_unslashed_attesting_indices(state, [a]): - non_proposer_attestations.append(a) - - assert any(non_proposer_attestations) - state.previous_epoch_attestations = non_proposer_attestations - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_duplicate_attestations_at_later_slots(spec, state): - prepare_state_with_full_attestations(spec, state) - - # Remove 2/3 of attestations to make it more interesting - num_attestations = int(len(state.previous_epoch_attestations) * 0.33) - state.previous_epoch_attestations = state.previous_epoch_attestations[:num_attestations] - - # Get map of the proposer at each slot to make valid-looking duplicate attestations - per_slot_proposers = { - (a.data.slot + a.inclusion_delay): a.proposer_index - for a in state.previous_epoch_attestations - } - max_slot = max([a.data.slot + a.inclusion_delay for a in state.previous_epoch_attestations]) - later_attestations = [] - for a in state.previous_epoch_attestations: - # Only have proposers for previous epoch so do not create later - # duplicate if slot exceeds the max slot in previous_epoch_attestations - if a.data.slot + a.inclusion_delay >= max_slot: - continue - later_a = a.copy() - later_a.inclusion_delay += 1 - later_a.proposer_index = per_slot_proposers[later_a.data.slot + later_a.inclusion_delay] - later_attestations.append(later_a) - - assert any(later_attestations) - - state.previous_epoch_attestations = sorted( - state.previous_epoch_attestations + later_attestations, - key=lambda a: a.data.slot + a.inclusion_delay - ) - - yield from run_get_inclusion_delay_deltas(spec, state) - - -@with_all_phases -@spec_state_test -def test_all_balances_too_low_for_reward(spec, state): - prepare_state_with_full_attestations(spec, state) - - for index in range(len(state.validators)): - state.validators[index].effective_balance = 10 - - yield from run_get_inclusion_delay_deltas(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py deleted file mode 100644 index 0ae9850866..0000000000 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_get_target_deltas.py +++ /dev/null @@ -1,128 +0,0 @@ -from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.rewards import run_attestation_component_deltas -import eth2spec.test.helpers.rewards as rewards_helpers - - -def run_get_target_deltas(spec, state): - """ - Run ``get_target_deltas``, yielding: - - pre-state ('pre') - - deltas ('deltas') - """ - - yield from run_attestation_component_deltas( - spec, - state, - spec.get_target_deltas, - spec.get_matching_target_attestations, - ) - - -@with_all_phases -@spec_state_test -def test_empty(spec, state): - yield from rewards_helpers.run_test_empty(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_full_all_correct(spec, state): - yield from rewards_helpers.run_test_full_all_correct(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_half_full(spec, state): - yield from rewards_helpers.run_test_half_full(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_full_but_partial_participation(spec, state): - yield from rewards_helpers.run_test_full_but_partial_participation(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_one_attestation_one_correct(spec, state): - yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_with_slashed_validators(spec, state): - yield from rewards_helpers.run_test_with_slashed_validators(spec, state, run_get_target_deltas) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_attested(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested( - spec, - state, - run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_some_very_low_effective_balances_that_did_not_attest(spec, state): - yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest( - spec, - state, - run_get_target_deltas, - ) - - -@with_all_phases -@spec_state_test -def test_full_half_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_correct_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=True, - correct_head=False, - fraction_incorrect=1.0, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_incorrect_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=False, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_half_incorrect_target_correct_head(spec, state): - yield from rewards_helpers.run_test_full_fraction_incorrect( - spec, state, - correct_target=False, - correct_head=True, - fraction_incorrect=0.5, - runner=run_get_target_deltas - ) - - -@with_all_phases -@spec_state_test -def test_full_random(spec, state): - yield from rewards_helpers.run_test_full_random(spec, state, run_get_target_deltas) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py new file mode 100644 index 0000000000..4e75079c05 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -0,0 +1,190 @@ +from eth2spec.test.context import with_all_phases, spec_state_test +from eth2spec.test.helpers.state import next_epoch +import eth2spec.test.helpers.rewards as rewards_helpers +from lru import LRU + + +def transition_state_to_leak(spec, state, epochs=None): + if epochs is None: + epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + + for _ in range(epochs): + next_epoch(spec, state) + + +_cache_dict = LRU(size=10) + + +def leaking(epochs=None): + + def deco(fn): + def entry(*args, spec, state, **kw): + # If the pre-state is not already known in the LRU, then take it, + # transition it to leak, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. + key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) + global _cache_dict + if key not in _cache_dict: + transition_state_to_leak(spec, state, epochs=epochs) + _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. + + # Take an entry out of the LRU. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_cache_dict[key]) + return fn(*args, spec=spec, state=state, **kw) + return entry + return deco + + +@with_all_phases +@spec_state_test +@leaking() +def test_empty_leak(spec, state): + yield from rewards_helpers.run_test_empty(spec, state) + + +@with_all_phases +@spec_state_test +@leaking() +def test_full_leak(spec, state): + yield from rewards_helpers.run_test_full_all_correct(spec, state) + + +@with_all_phases +@spec_state_test +@leaking() +def test_half_full_leak(spec, state): + yield from rewards_helpers.run_test_half_full(spec, state) + + +@with_all_phases +@spec_state_test +@leaking() +def test_quarter_full_leak(spec, state): + yield from rewards_helpers.run_test_partial(spec, state, 0.25) + + +@with_all_phases +@spec_state_test +@leaking() +def test_full_but_partial_participation_leak(spec, state): + yield from rewards_helpers.run_test_full_but_partial_participation(spec, state) + + +@with_all_phases +@spec_state_test +@leaking() +def test_one_attestation_one_correct_leak(spec, state): + yield from rewards_helpers.run_test_one_attestation_one_correct(spec, state) + + +@with_all_phases +@spec_state_test +@leaking() +def test_with_not_yet_activated_validators_leak(spec, state): + yield from rewards_helpers.run_test_with_not_yet_activated_validators(spec, state) + + +@with_all_phases +@spec_state_test +@leaking() +def test_with_exited_validators_leak(spec, state): + yield from rewards_helpers.run_test_with_exited_validators(spec, state) + + +@with_all_phases +@spec_state_test +@leaking() +def test_with_slashed_validators_leak(spec, state): + yield from rewards_helpers.run_test_with_slashed_validators(spec, state) + + +@with_all_phases +@spec_state_test +@leaking() +def test_some_very_low_effective_balances_that_attested_leak(spec, state): + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_attested(spec, state) + + +@with_all_phases +@spec_state_test +@leaking() +def test_some_very_low_effective_balances_that_did_not_attest_leak(spec, state): + yield from rewards_helpers.run_test_some_very_low_effective_balances_that_did_not_attest(spec, state) + + +# +# NOTE: No source incorrect tests +# All PendingAttestations in state have source validated +# We choose to keep this invariant in these tests to not force clients to test with degenerate states +# + + +@with_all_phases +@spec_state_test +@leaking() +def test_full_half_correct_target_incorrect_head_leak(spec, state): + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +@leaking() +def test_full_correct_target_incorrect_head_leak(spec, state): + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=True, + correct_head=False, + fraction_incorrect=1.0, + ) + + +@with_all_phases +@spec_state_test +@leaking() +def test_full_half_incorrect_target_incorrect_head_leak(spec, state): + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=False, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +@leaking() +def test_full_half_incorrect_target_correct_head_leak(spec, state): + yield from rewards_helpers.run_test_full_fraction_incorrect( + spec, state, + correct_target=False, + correct_head=True, + fraction_incorrect=0.5, + ) + + +@with_all_phases +@spec_state_test +@leaking() +def test_full_random_leak(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@spec_state_test +@leaking(epochs=5) +def test_full_random_five_epoch_leak(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@spec_state_test +@leaking(epochs=10) +def test_full_random_ten_epoch_leak(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py new file mode 100644 index 0000000000..83c7f79051 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_random.py @@ -0,0 +1,45 @@ +from random import Random + +from eth2spec.test.context import ( + with_all_phases, + spec_test, + spec_state_test, + with_custom_state, + single_phase, + low_balances, misc_balances, +) +import eth2spec.test.helpers.rewards as rewards_helpers + + +@with_all_phases +@spec_state_test +def test_full_random_0(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(1010)) + + +@with_all_phases +@spec_state_test +def test_full_random_1(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(2020)) + + +@with_all_phases +@spec_state_test +def test_full_random_2(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state, rng=Random(3030)) + + +@with_all_phases +@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_full_random_low_balances(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) + + +@with_all_phases +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +def test_full_random_misc_balances(spec, state): + yield from rewards_helpers.run_test_full_random(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index a289d3b64a..f0cfc462e6 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -10,7 +10,11 @@ transition_unsigned_block, ) from eth2spec.test.helpers.keys import privkeys, pubkeys -from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing, get_indexed_attestation_participants +from eth2spec.test.helpers.attester_slashings import ( + get_valid_attester_slashing_by_indices, + get_valid_attester_slashing, + get_indexed_attestation_participants, +) from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations from eth2spec.test.helpers.deposits import prepare_state_and_deposit @@ -409,13 +413,14 @@ def test_multiple_different_proposer_slashings_same_block(spec, state): check_proposer_slashing_effect(spec, pre_state, state, slashed_index) -def check_attester_slashing_effect(spec, pre_state, state, validator_index): - slashed_validator = state.validators[validator_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) +def check_attester_slashing_effect(spec, pre_state, state, slashed_indices): + for slashed_index in slashed_indices: + slashed_validator = state.validators[slashed_index] + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + assert get_balance(state, slashed_index) < get_balance(pre_state, slashed_index) proposer_index = spec.get_beacon_proposer_index(state) # gained whistleblower reward @@ -429,9 +434,9 @@ def test_attester_slashing(spec, state): pre_state = state.copy() attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - validator_index = get_indexed_attestation_participants(spec, attester_slashing.attestation_1)[0] + slashed_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) - assert not state.validators[validator_index].slashed + assert not any(state.validators[i].slashed for i in slashed_indices) yield 'pre', state @@ -446,11 +451,118 @@ def test_attester_slashing(spec, state): yield 'blocks', [signed_block] yield 'post', state - check_attester_slashing_effect(spec, pre_state, state, validator_index) + check_attester_slashing_effect(spec, pre_state, state, slashed_indices) + + +@with_all_phases +@spec_state_test +def test_duplicate_attester_slashing(spec, state): + # Skip test if config cannot handle multiple AttesterSlashings per block + if spec.MAX_ATTESTER_SLASHINGS < 2: + return + + attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) + attester_slashings = [attester_slashing, attester_slashing.copy()] + slashed_indices = get_indexed_attestation_participants(spec, attester_slashing.attestation_1) + + assert not any(state.validators[i].slashed for i in slashed_indices) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.attester_slashings = attester_slashings + + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +# All AttesterSlashing tests should be adopted for Phase 1 but helper support is not yet there + +@with_phases(['phase0']) +@spec_state_test +def test_multiple_attester_slashings_no_overlap(spec, state): + # Skip test if config cannot handle multiple AttesterSlashings per block + if spec.MAX_ATTESTER_SLASHINGS < 2: + return + + # copy for later balance lookups. + pre_state = state.copy() + + full_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[:8] + half_length = len(full_indices) // 2 + + attester_slashing_1 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[:half_length], signed_1=True, signed_2=True, + ) + attester_slashing_2 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[half_length:], signed_1=True, signed_2=True, + ) + attester_slashings = [attester_slashing_1, attester_slashing_2] + + assert not any(state.validators[i].slashed for i in full_indices) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.attester_slashings = attester_slashings + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state + + check_attester_slashing_effect(spec, pre_state, state, full_indices) + + +@with_phases(['phase0']) +@spec_state_test +def test_multiple_attester_slashings_partial_overlap(spec, state): + # Skip test if config cannot handle multiple AttesterSlashings per block + if spec.MAX_ATTESTER_SLASHINGS < 2: + return + + # copy for later balance lookups. + pre_state = state.copy() + + full_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[:8] + one_third_length = len(full_indices) // 3 + + attester_slashing_1 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[:one_third_length * 2], signed_1=True, signed_2=True, + ) + attester_slashing_2 = get_valid_attester_slashing_by_indices( + spec, state, + full_indices[one_third_length:], signed_1=True, signed_2=True, + ) + attester_slashings = [attester_slashing_1, attester_slashing_2] + + assert not any(state.validators[i].slashed for i in full_indices) + + yield 'pre', state + + # + # Add to state via block transition + # + block = build_empty_block_for_next_slot(spec, state) + block.body.attester_slashings = attester_slashings + + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state -# TODO: currently mainnet limits attester-slashings per block to 1. -# When this is increased, it should be tested to cover various combinations -# of duplicate slashings and overlaps of slashed attestations within the same block + check_attester_slashing_effect(spec, pre_state, state, full_indices) @with_all_phases @@ -614,7 +726,7 @@ def create_signed_exit(index): return [create_signed_exit(index) for index in indices] -# In phase1 a committee is computed for PERSISTENT_COMMITTEE_PERIOD slots ago, +# In phase1 a committee is computed for SHARD_COMMITTEE_PERIOD slots ago, # exceeding the minimal-config randao mixes memory size. # Applies to all voluntary-exit sanity block tests. @@ -623,8 +735,8 @@ def create_signed_exit(index): def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH signed_exits = prepare_signed_exits(spec, state, [validator_index]) yield 'pre', state @@ -651,8 +763,8 @@ def test_voluntary_exit(spec, state): def test_double_validator_exit_same_block(spec, state): validator_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Same index tries to exit twice, but should only be able to do so once. signed_exits = prepare_signed_exits(spec, state, [validator_index, validator_index]) @@ -674,8 +786,8 @@ def test_multiple_different_validator_exits_same_block(spec, state): spec.get_active_validator_indices(state, spec.get_current_epoch(state))[i] for i in range(3) ] - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH signed_exits = prepare_signed_exits(spec, state, validator_indices) yield 'pre', state diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py new file mode 100644 index 0000000000..5bb246ed5c --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -0,0 +1,395 @@ +from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.helpers.attestations import build_attestation_data +from eth2spec.test.helpers.block import build_empty_block +from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.test.helpers.state import next_epoch +from eth2spec.utils import bls +from eth2spec.utils.ssz.ssz_typing import Bitlist + + +def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey): + signature = get_signature_fn(state, obj, privkey) + signing_root = spec.compute_signing_root(obj, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +def run_get_committee_assignment(spec, state, epoch, validator_index, valid=True): + try: + assignment = spec.get_committee_assignment(state, epoch, validator_index) + committee, committee_index, slot = assignment + assert spec.compute_epoch_at_slot(slot) == epoch + assert committee == spec.get_beacon_committee(state, slot, committee_index) + assert committee_index < spec.get_committee_count_at_slot(state, slot) + assert validator_index in committee + assert valid + except AssertionError: + assert not valid + else: + assert valid + + +def run_is_candidate_block(spec, eth1_block, period_start, success=True): + assert success == spec.is_candidate_block(eth1_block, period_start) + + +def get_min_new_period_epochs(spec): + return int( + spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 # to seconds + / spec.SECONDS_PER_SLOT / spec.SLOTS_PER_EPOCH + ) + + +def get_mock_aggregate(spec): + return spec.Attestation( + data=spec.AttestationData( + slot=10, + ) + ) + + +# +# Becoming a validator +# + + +@with_all_phases +@spec_state_test +@never_bls +def test_check_if_validator_active(spec, state): + active_validator_index = len(state.validators) - 1 + assert spec.check_if_validator_active(state, active_validator_index) + new_validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit = prepare_state_and_deposit(spec, state, new_validator_index, amount, signed=True) + spec.process_deposit(state, deposit) + assert not spec.check_if_validator_active(state, new_validator_index) + + +# +# Validator assignments +# + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment_current_epoch(spec, state): + epoch = spec.get_current_epoch(state) + validator_index = len(state.validators) - 1 + run_get_committee_assignment(spec, state, epoch, validator_index, valid=True) + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment_next_epoch(spec, state): + epoch = spec.get_current_epoch(state) + 1 + validator_index = len(state.validators) - 1 + run_get_committee_assignment(spec, state, epoch, validator_index, valid=True) + + +@with_all_phases +@spec_state_test +@never_bls +def test_get_committee_assignment_out_bound_epoch(spec, state): + epoch = spec.get_current_epoch(state) + 2 + validator_index = len(state.validators) - 1 + run_get_committee_assignment(spec, state, epoch, validator_index, valid=False) + + +@with_all_phases +@spec_state_test +@never_bls +def test_is_proposer(spec, state): + proposer_index = spec.get_beacon_proposer_index(state) + assert spec.is_proposer(state, proposer_index) + + proposer_index = proposer_index + 1 % len(state.validators) + assert not spec.is_proposer(state, proposer_index) + + +# +# Beacon chain responsibilities +# + + +# Block proposal + + +@with_all_phases +@spec_state_test +def test_get_epoch_signature(spec, state): + block = spec.BeaconBlock() + privkey = privkeys[0] + pubkey = pubkeys[0] + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) + run_get_signature_test( + spec=spec, + state=state, + obj=block, + domain=domain, + get_signature_fn=spec.get_epoch_signature, + privkey=privkey, + pubkey=pubkey, + ) + + +@with_all_phases +@spec_state_test +def test_is_candidate_block(spec, state): + period_start = spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 + 1000 + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE), + period_start, + success=True, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE + 1), + period_start, + success=False, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2), + period_start, + success=True, + ) + run_is_candidate_block( + spec, + spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE * 2 - 1), + period_start, + success=False, + ) + + +@with_all_phases +@spec_state_test +def test_get_eth1_vote_default_vote(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs): + next_epoch(spec, state) + + state.eth1_data_votes = () + eth1_chain = [] + eth1_data = spec.get_eth1_vote(state, eth1_chain) + assert eth1_data == state.eth1_data + + +@with_all_phases +@spec_state_test +def test_get_eth1_vote_consensus_vote(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs + 2): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length >= 3 # We need to have the majority vote + state.eth1_data_votes = () + + block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) + block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + eth1_chain = [block_1, block_2] + eth1_data_votes = [] + + # Only the first vote is for block_1 + eth1_data_votes.append(spec.get_eth1_data(block_1)) + # Other votes are for block_2 + for _ in range(votes_length - 1): + eth1_data_votes.append(spec.get_eth1_data(block_2)) + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + assert eth1_data.block_hash == block_2.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_get_eth1_vote_tie(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs + 1): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length > 0 and votes_length % 2 == 0 + + state.eth1_data_votes = () + block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) + block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + eth1_chain = [block_1, block_2] + eth1_data_votes = [] + # Half votes are for block_1, another half votes are for block_2 + for i in range(votes_length): + if i % 2 == 0: + block = block_1 + else: + block = block_2 + eth1_data_votes.append(spec.get_eth1_data(block)) + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + + # Tiebreak by smallest distance -> eth1_chain[0] + assert eth1_data.block_hash == eth1_chain[0].hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_compute_new_state_root(spec, state): + pre_state = state.copy() + post_state = state.copy() + block = build_empty_block(spec, state, state.slot + 1) + state_root = spec.compute_new_state_root(state, block) + + assert state_root != pre_state.hash_tree_root() + assert state == pre_state + + # dumb verification + spec.process_slots(post_state, block.slot) + spec.process_block(post_state, block) + assert state_root == post_state.hash_tree_root() + + +@with_all_phases +@spec_state_test +def test_get_block_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + block = build_empty_block(spec, state) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) + run_get_signature_test( + spec=spec, + state=state, + obj=block, + domain=domain, + get_signature_fn=spec.get_block_signature, + privkey=privkey, + pubkey=pubkey, + ) + + +# Attesting + + +@with_all_phases +@spec_state_test +def test_get_attestation_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + attestation_data = spec.AttestationData(slot=10) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + run_get_signature_test( + spec=spec, + state=state, + obj=attestation_data, + domain=domain, + get_signature_fn=spec.get_attestation_signature, + privkey=privkey, + pubkey=pubkey, + ) + + +# Attestation aggregation + + +@with_all_phases +@spec_state_test +def test_get_slot_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + slot = spec.Slot(10) + domain = spec.get_domain(state, spec.DOMAIN_SELECTION_PROOF, spec.compute_epoch_at_slot(slot)) + run_get_signature_test( + spec=spec, + state=state, + obj=slot, + domain=domain, + get_signature_fn=spec.get_slot_signature, + privkey=privkey, + pubkey=pubkey, + ) + + +@with_all_phases +@spec_state_test +def test_is_aggregator(spec, state): + # TODO: we can test the probabilistic result against `TARGET_AGGREGATORS_PER_COMMITTEE` + # if we have more validators and larger committeee size + slot = state.slot + committee_index = 0 + has_aggregator = False + beacon_committee = spec.get_beacon_committee(state, slot, committee_index) + for validator_index in beacon_committee: + privkey = privkeys[validator_index] + slot_signature = spec.get_slot_signature(state, slot, privkey) + if spec.is_aggregator(state, slot, committee_index, slot_signature): + has_aggregator = True + break + assert has_aggregator + + +@with_all_phases +@spec_state_test +def test_get_aggregate_signature(spec, state): + attestations = [] + pubkeys = [] + slot = state.slot + committee_index = 0 + attestation_data = build_attestation_data(spec, state, slot=slot, index=committee_index) + beacon_committee = spec.get_beacon_committee( + state, + attestation_data.slot, + attestation_data.index, + ) + committee_size = len(beacon_committee) + aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) + for i, validator_index in enumerate(beacon_committee): + bits = aggregation_bits + bits[i] = True + attestations.append( + spec.Attestation( + data=attestation_data, + aggregation_bits=bits, + ) + ) + pubkeys.append(state.validators[validator_index].pubkey) + pubkey = bls.AggregatePKs(pubkeys) + signature = spec.get_aggregate_signature(attestations) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = spec.compute_signing_root(attestation_data, domain) + assert bls.Verify(pubkey, signing_root, signature) + + +@with_all_phases +@spec_state_test +def test_get_aggregate_and_proof(spec, state): + privkey = privkeys[0] + aggregator_index = spec.ValidatorIndex(10) + aggregate = get_mock_aggregate(spec) + aggregate_and_proof = spec.get_aggregate_and_proof(state, aggregator_index, aggregate, privkey) + assert aggregate_and_proof.aggregator_index == aggregator_index + assert aggregate_and_proof.aggregate == aggregate + assert aggregate_and_proof.selection_proof == spec.get_slot_signature(state, aggregate.data.slot, privkey) + + +@with_all_phases +@spec_state_test +def test_get_aggregate_and_proof_signature(spec, state): + privkey = privkeys[0] + pubkey = pubkeys[0] + aggregate = get_mock_aggregate(spec) + aggregate_and_proof = spec.get_aggregate_and_proof(state, spec.ValidatorIndex(1), aggregate, privkey) + domain = spec.get_domain(state, spec.DOMAIN_AGGREGATE_AND_PROOF, spec.compute_epoch_at_slot(aggregate.data.slot)) + run_get_signature_test( + spec=spec, + state=state, + obj=aggregate_and_proof, + domain=domain, + get_signature_fn=spec.get_aggregate_and_proof_signature, + privkey=privkey, + pubkey=pubkey, + ) diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index 83371ac628..acf9f99c7d 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -25,17 +25,32 @@ def entry(*args, **kw): @only_with_bls(alt_return=True) def Verify(PK, message, signature): - return bls.Verify(PK, message, signature) + try: + result = bls.Verify(PK, message, signature) + except Exception: + result = False + finally: + return result @only_with_bls(alt_return=True) -def AggregateVerify(pairs, signature): - return bls.AggregateVerify(pairs, signature) +def AggregateVerify(pubkeys, messages, signature): + try: + result = bls.AggregateVerify(pubkeys, messages, signature) + except Exception: + result = False + finally: + return result @only_with_bls(alt_return=True) -def FastAggregateVerify(PKs, message, signature): - return bls.FastAggregateVerify(PKs, message, signature) +def FastAggregateVerify(pubkeys, message, signature): + try: + result = bls.FastAggregateVerify(pubkeys, message, signature) + except Exception: + result = False + finally: + return result @only_with_bls(alt_return=STUB_SIGNATURE) @@ -51,3 +66,13 @@ def Sign(SK, message): @only_with_bls(alt_return=STUB_COORDINATES) def signature_to_G2(signature): return _signature_to_G2(signature) + + +@only_with_bls(alt_return=STUB_PUBKEY) +def AggregatePKs(pubkeys): + return bls._AggregatePKs(pubkeys) + + +@only_with_bls(alt_return=STUB_SIGNATURE) +def SkToPk(SK): + return bls.SkToPk(SK) diff --git a/tests/core/pyspec/eth2spec/utils/hash_function.py b/tests/core/pyspec/eth2spec/utils/hash_function.py index 2c9b5a579a..627f9b9904 100644 --- a/tests/core/pyspec/eth2spec/utils/hash_function.py +++ b/tests/core/pyspec/eth2spec/utils/hash_function.py @@ -1,28 +1,17 @@ from hashlib import sha256 +from typing import Dict, Union ZERO_BYTES32 = b'\x00' * 32 -def _hash(x): +def _hash(x: Union[bytes, bytearray, memoryview]) -> bytes: return sha256(x).digest() -# Minimal collection of (key, value) pairs, for fast hash-retrieval, to save on repetitive computation cost. -# Key = the hash input -# Value = the hash output -hash_cache = [] +hash_cache: Dict[bytes, bytes] = {} -def add_zero_hashes_to_cache(): - zerohashes = [(None, ZERO_BYTES32)] - for layer in range(1, 32): - k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] - zerohashes.append((k, _hash(k))) - hash_cache.extend(zerohashes[1:]) - - -def hash(x): - for (k, h) in hash_cache: - if x == k: - return h +def hash(x: bytes) -> bytes: + if x in hash_cache: + return hash_cache[x] return _hash(x) diff --git a/tests/formats/bls/README.md b/tests/formats/bls/README.md index 4d95bdfd77..65154ba1cf 100644 --- a/tests/formats/bls/README.md +++ b/tests/formats/bls/README.md @@ -5,11 +5,10 @@ We do not recommend rolling your own crypto or using an untested BLS library. The BLS test suite runner has the following handlers: -- [`aggregate_pubkeys`](./aggregate_pubkeys.md) -- [`aggregate_sigs`](./aggregate_sigs.md) -- [`msg_hash_g2_compressed`](./msg_hash_g2_compressed.md) -- [`msg_hash_g2_uncompressed`](./msg_hash_g2_uncompressed.md) -- [`priv_to_pub`](./priv_to_pub.md) -- [`sign_msg`](./sign_msg.md) +- [`aggregate_verify`](./aggregate_verify.md) +- [`aggregate`](./aggregate.md) +- [`fast_aggregate_verify`](./fast_aggregate_verify.md) +- [`sign`](./sign.md) +- [`verify`](./verify.md) *Note*: Signature-verification and aggregate-verify test cases are not yet supported. diff --git a/tests/formats/bls/aggregate.md b/tests/formats/bls/aggregate.md new file mode 100644 index 0000000000..af8444540c --- /dev/null +++ b/tests/formats/bls/aggregate.md @@ -0,0 +1,19 @@ +# Test format: BLS signature aggregation + +A BLS signature aggregation combines a series of signatures into a single signature. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: List[BLS Signature] -- list of input BLS signatures +output: BLS Signature -- expected output, single BLS signature or empty. +``` + +- `BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`. +- No output value if the input is invalid. + +## Condition + +The `aggregate` handler should aggregate the signatures in the `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/aggregate_pubkeys.md b/tests/formats/bls/aggregate_pubkeys.md deleted file mode 100644 index 049ad6991b..0000000000 --- a/tests/formats/bls/aggregate_pubkeys.md +++ /dev/null @@ -1,19 +0,0 @@ -# Test format: BLS pubkey aggregation - -A BLS pubkey aggregation combines a series of pubkeys into a single pubkey. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: List[BLS Pubkey] -- list of input BLS pubkeys -output: BLS Pubkey -- expected output, single BLS pubkey -``` - -`BLS Pubkey` here is encoded as a string: hexadecimal encoding of 48 bytes (96 nibbles), prefixed with `0x`. - - -## Condition - -The `aggregate_pubkeys` handler should aggregate the keys in the `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/aggregate_sigs.md b/tests/formats/bls/aggregate_sigs.md deleted file mode 100644 index 2252dbaa80..0000000000 --- a/tests/formats/bls/aggregate_sigs.md +++ /dev/null @@ -1,19 +0,0 @@ -# Test format: BLS signature aggregation - -A BLS signature aggregation combines a series of signatures into a single signature. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: List[BLS Signature] -- list of input BLS signatures -output: BLS Signature -- expected output, single BLS signature -``` - -`BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`. - - -## Condition - -The `aggregate_sigs` handler should aggregate the signatures in the `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/aggregate_verify.md b/tests/formats/bls/aggregate_verify.md new file mode 100644 index 0000000000..3985de9f43 --- /dev/null +++ b/tests/formats/bls/aggregate_verify.md @@ -0,0 +1,17 @@ +# Test format: BLS sign message + +Verify the signature against the given pubkeys and one messages. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: + pubkeys: List[bytes48] -- the pubkeys + messages: List[bytes32] -- the messages + signature: bytes96 -- the signature to verify against pubkeys and messages +output: bool -- VALID or INVALID +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/bls/fast_aggregate_verify.md b/tests/formats/bls/fast_aggregate_verify.md new file mode 100644 index 0000000000..7e3899a15f --- /dev/null +++ b/tests/formats/bls/fast_aggregate_verify.md @@ -0,0 +1,17 @@ +# Test format: BLS sign message + +Verify the signature against the given pubkeys and one message. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: + pubkeys: List[bytes48] -- the pubkey + message: bytes32 -- the message + signature: bytes96 -- the signature to verify against pubkeys and message +output: bool -- VALID or INVALID +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/bls/msg_hash_g2_compressed.md b/tests/formats/bls/msg_hash_g2_compressed.md deleted file mode 100644 index 761e819f29..0000000000 --- a/tests/formats/bls/msg_hash_g2_compressed.md +++ /dev/null @@ -1,21 +0,0 @@ -# Test format: BLS hash-compressed - -A BLS compressed-hash to G2. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: - message: bytes32 - domain: bytes8 -- the BLS domain -output: List[bytes48] -- length of two -``` - -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `msg_hash_g2_compressed` handler should hash the `message`, with the given `domain`, to G2 with compression, and the result should match the expected `output`. diff --git a/tests/formats/bls/msg_hash_g2_uncompressed.md b/tests/formats/bls/msg_hash_g2_uncompressed.md deleted file mode 100644 index 5ee535a38e..0000000000 --- a/tests/formats/bls/msg_hash_g2_uncompressed.md +++ /dev/null @@ -1,21 +0,0 @@ -# Test format: BLS hash-uncompressed - -A BLS uncompressed-hash to G2. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: - message: bytes32 - domain: bytes8 -- the BLS domain -output: List[List[bytes48]] -- 3 lists, each a length of two -``` - -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `msg_hash_g2_uncompressed` handler should hash the `message`, with the given `domain`, to G2, without compression, and the result should match the expected `output`. diff --git a/tests/formats/bls/priv_to_pub.md b/tests/formats/bls/priv_to_pub.md deleted file mode 100644 index 29c6b216a1..0000000000 --- a/tests/formats/bls/priv_to_pub.md +++ /dev/null @@ -1,19 +0,0 @@ -# Test format: BLS private key to pubkey - -A BLS private key to public key conversion. - -## Test case format - -The test data is declared in a `data.yaml` file: - -```yaml -input: bytes32 -- the private key -output: bytes48 -- the public key -``` - -All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `priv_to_pub` handler should compute the public key for the given private key `input`, and the result should match the expected `output`. diff --git a/tests/formats/bls/sign_msg.md b/tests/formats/bls/sign.md similarity index 67% rename from tests/formats/bls/sign_msg.md rename to tests/formats/bls/sign.md index 6c4f88cd1b..1c328755a5 100644 --- a/tests/formats/bls/sign_msg.md +++ b/tests/formats/bls/sign.md @@ -10,13 +10,7 @@ The test data is declared in a `data.yaml` file: input: privkey: bytes32 -- the private key used for signing message: bytes32 -- input message to sign (a hash) - domain: bytes8 -- the BLS domain output: bytes96 -- expected signature ``` All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. - - -## Condition - -The `sign_msg` handler should sign the given `message`, with `domain`, using the given `privkey`, and the result should match the expected `output`. diff --git a/tests/formats/bls/verify.md b/tests/formats/bls/verify.md new file mode 100644 index 0000000000..57ec8a33a7 --- /dev/null +++ b/tests/formats/bls/verify.md @@ -0,0 +1,17 @@ +# Test format: BLS sign message + +Verify the signature against the given one pubkey and one message. + +## Test case format + +The test data is declared in a `data.yaml` file: + +```yaml +input: + pubkey: bytes48 -- the pubkey + message: bytes32 -- the message + signature: bytes96 -- the signature to verify against pubkey and message +output: bool -- VALID or INVALID +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md index 7c5e2dc701..57c9441c8d 100644 --- a/tests/formats/epoch_processing/README.md +++ b/tests/formats/epoch_processing/README.md @@ -38,7 +38,7 @@ The provided pre-state is already transitioned to just before the specific sub-t Sub-transitions: - `justification_and_finalization` -- *`rewards_and_penalties` - planned testing extension* +- `rewards_and_penalties` (limited to `minimal` config) - `registry_updates` - `slashings` - `final_updates` diff --git a/tests/formats/rewards/README.md b/tests/formats/rewards/README.md index f70a20f9c2..b229d9f985 100644 --- a/tests/formats/rewards/README.md +++ b/tests/formats/rewards/README.md @@ -1,8 +1,15 @@ # Rewards tests -The different rewards deltas sub-functions are testing individually with the test handlers, each returning the related `rewards`/`penalties`. +All rewards deltas sub-functions are tested for each test case. There is no "change" factor, the rewards/penalties outputs are pure functions with just the pre-state as input. -Hence, the format is shared between each test-handler. (See test condition documentation on how to run the tests.) +(See test condition documentation on how to run the tests.) + +`Deltas` is defined as: +```python +class Deltas(Container): + rewards: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + penalties: List[Gwei, VALIDATOR_REGISTRY_LIMIT] +``` ## Test case format @@ -22,31 +29,47 @@ A YAML-encoded `BeaconState`, the state before running the rewards sub-function. Also available as `pre.ssz`. -### `deltas.yaml` +### `source_deltas.yaml` -A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards sub-function +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_source_deltas` function -Where `Deltas` is defined as: -```python -class Deltas(Container): - rewards: List[uint64, VALIDATOR_REGISTRY_LIMIT] - penalties: List[uint64, VALIDATOR_REGISTRY_LIMIT] -``` +Also available as `source_deltas.ssz`. + +### `target_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_target_deltas` function + +Also available as `target_deltas.ssz`. + +### `head_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_head_deltas` function + +Also available as `head_deltas.ssz`. + +### `inclusion_delay_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inclusion_delay_deltas` function + +Also available as `inclusion_delay_deltas.ssz`. + +### `inactivity_penalty_deltas.yaml` + +A YAML-encoded `Deltas` representing the rewards and penalties returned by the rewards the `get_inactivity_penalty_deltas` function -Also available as `deltas.ssz`. +Also available as `inactivity_penalty_deltas.ssz`. ## Condition A handler of the `rewards` test-runner should process these cases, - calling the corresponding rewards deltas function (same name in spec). -This excludes all other parts of `process_rewards_and_penalties` + calling the corresponding rewards deltas function for each set of deltas. -The provided pre-state is ready to be input into the designated handler. +The provided pre-state is ready to be input into each rewards deltas function. The provided `deltas` should match the return values of the - handler. Specifically the following must hold true: + deltas function. Specifically the following must hold true for each set of deltas: ```python - deltas.rewards == handler(state)[0] - deltas.penalties == handler(state)[1] + deltas.rewards == deltas_function(state)[0] + deltas.penalties == deltas_function(state)[1] ``` diff --git a/tests/generators/bls/README.md b/tests/generators/bls/README.md index 878bb156ba..24013f88e7 100644 --- a/tests/generators/bls/README.md +++ b/tests/generators/bls/README.md @@ -1,21 +1,11 @@ # BLS Test Generator -Explanation of BLS12-381 type hierarchy -The base unit is bytes48 of which only 381 bits are used +The [BLS Signature APIs](../../../specs/phase0/beacon-chain.md#bls-signatures) -- FQ: uint381 modulo field modulus -- FQ2: (FQ, FQ) -- G2: (FQ2, FQ2, FQ2) +Information on the format of the tests can be found in the [BLS test formats documentation](../../formats/bls/README.md). ## Resources -- [Eth2 spec](../../../specs/phase0/beacon-chain.md#bls-signatures) +- [IETF BLS Signature Scheme](https://datatracker.ietf.org/doc/draft-irtf-cfrg-bls-signature/) - [Finite Field Arithmetic](http://www.springeronline.com/sgw/cda/pageitems/document/cda_downloaddocument/0,11996,0-0-45-110359-0,00.pdf) -- Chapter 2 of [Elliptic Curve Cryptography](http://cacr.uwaterloo.ca/ecc/). Darrel Hankerson, Alfred Menezes, and Scott Vanstone -- [Zcash BLS parameters](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381) -- [Trinity implementation](https://github.com/ethereum/trinity/blob/master/eth2/_utils/bls.py) - -## Comments - -Compared to Zcash, Ethereum specs always requires the compressed form (c_flag / most significant bit always set). -Also note that pubkeys and privkeys are reversed. +- Chapter 2 of [Elliptic Curve Cryptography](http://cacr.uwaterloo.ca/ecc/). Darrel Hankerson, Alfred Menezes, and Scott Vanstone diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 455292ae38..8c6589b364 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -10,20 +10,16 @@ ) from gen_base import gen_runner, gen_typing -from py_ecc import bls +from eth2spec.utils import bls from hashlib import sha256 from eth2spec.test.context import PHASE0 + def hash(x): return sha256(x).digest() -F2Q_COEFF_LEN = 48 -G2_COMPRESSED_Z_LEN = 48 -DST = bls.G2ProofOfPossession.DST - - def int_to_hex(n: int, byte_length: int = None) -> str: byte_value = int_to_big_endian(n) if byte_length: @@ -49,11 +45,15 @@ def hex_to_int(x: str) -> int: hex_to_int('0x00000000000000000000000000000000328388aff0d4a5b7dc9205abd374e7e98f3cd9f3418edb4eafda5fb16473d216'), ] +Z1_PUBKEY = b'\xc0' + b'\x00' * 47 +NO_SIGNATURE = b'\x00' * 96 +Z2_SIGNATURE = b'\xc0' + b'\x00' * 95 + def case01_sign(): for privkey in PRIVKEYS: for message in MESSAGES: - sig = bls.G2ProofOfPossession.Sign(privkey, message) + sig = bls.Sign(privkey, message) identifier = f'{int_to_hex(privkey)}_{encode_hex(message)}' yield f'sign_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { @@ -68,9 +68,10 @@ def case02_verify(): for i, privkey in enumerate(PRIVKEYS): for message in MESSAGES: # Valid signature - signature = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.PrivToPub(privkey) + signature = bls.Sign(privkey, message) + pubkey = bls.SkToPk(privkey) identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' + assert bls.Verify(pubkey, message, signature) yield f'verify_valid_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -81,8 +82,9 @@ def case02_verify(): } # Invalid signatures -- wrong pubkey - wrong_pubkey = bls.G2ProofOfPossession.PrivToPub(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) + wrong_pubkey = bls.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) identifier = f'{encode_hex(wrong_pubkey)}_{encode_hex(message)}' + assert not bls.Verify(wrong_pubkey, message, signature) yield f'verify_wrong_pubkey_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(wrong_pubkey), @@ -95,6 +97,7 @@ def case02_verify(): # Invalid signature -- tampered with signature tampered_signature = signature[:-4] + b'\xFF\xFF\xFF\xFF' identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' + assert not bls.Verify(pubkey, message, tampered_signature) yield f'verify_tampered_signature_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -104,26 +107,51 @@ def case02_verify(): 'output': False, } + # Valid pubkey and signature with the point at infinity + assert bls.Verify(Z1_PUBKEY, message, Z2_SIGNATURE) + yield f'verify_infinity_pubkey_and_infinity_signature', { + 'input': { + 'pubkey': encode_hex(Z1_PUBKEY), + 'message': encode_hex(message), + 'signature': encode_hex(Z2_SIGNATURE), + }, + 'output': True, + } + def case03_aggregate(): for message in MESSAGES: - sigs = [bls.G2ProofOfPossession.Sign(privkey, message) for privkey in PRIVKEYS] + sigs = [bls.Sign(privkey, message) for privkey in PRIVKEYS] yield f'aggregate_{encode_hex(message)}', { 'input': [encode_hex(sig) for sig in sigs], - 'output': encode_hex(bls.G2ProofOfPossession.Aggregate(sigs)), + 'output': encode_hex(bls.Aggregate(sigs)), } + # Invalid pubkeys -- len(pubkeys) == 0 + try: + bls.Aggregate([]) + except Exception: + pass + else: + raise Exception("Should have been INVALID") + + yield f'aggregate_na_pubkeys', { + 'input': [], + 'output': None, + } + def case04_fast_aggregate_verify(): for i, message in enumerate(MESSAGES): privkeys = PRIVKEYS[:i + 1] - sigs = [bls.G2ProofOfPossession.Sign(privkey, message) for privkey in privkeys] - aggregate_signature = bls.G2ProofOfPossession.Aggregate(sigs) - pubkeys = [bls.G2ProofOfPossession.PrivToPub(privkey) for privkey in privkeys] + sigs = [bls.Sign(privkey, message) for privkey in privkeys] + aggregate_signature = bls.Aggregate(sigs) + pubkeys = [bls.SkToPk(privkey) for privkey in privkeys] pubkeys_serial = [encode_hex(pubkey) for pubkey in pubkeys] # Valid signature identifier = f'{pubkeys_serial}_{encode_hex(message)}' + assert bls.FastAggregateVerify(pubkeys, message, aggregate_signature) yield f'fast_aggregate_verify_valid_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -134,9 +162,10 @@ def case04_fast_aggregate_verify(): } # Invalid signature -- extra pubkey - pubkeys_extra = pubkeys + [bls.G2ProofOfPossession.PrivToPub(PRIVKEYS[-1])] + pubkeys_extra = pubkeys + [bls.SkToPk(PRIVKEYS[-1])] pubkeys_extra_serial = [encode_hex(pubkey) for pubkey in pubkeys_extra] identifier = f'{pubkeys_extra_serial}_{encode_hex(message)}' + assert not bls.FastAggregateVerify(pubkeys_extra, message, aggregate_signature) yield f'fast_aggregate_verify_extra_pubkey_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_extra_serial, @@ -149,6 +178,7 @@ def case04_fast_aggregate_verify(): # Invalid signature -- tampered with signature tampered_signature = aggregate_signature[:-4] + b'\xff\xff\xff\xff' identifier = f'{pubkeys_serial}_{encode_hex(message)}' + assert not bls.FastAggregateVerify(pubkeys, message, tampered_signature) yield f'fast_aggregate_verify_tampered_signature_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -158,37 +188,88 @@ def case04_fast_aggregate_verify(): 'output': False, } + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE + assert not bls.FastAggregateVerify([], message, Z2_SIGNATURE) + yield f'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { + 'input': { + 'pubkeys': [], + 'message': encode_hex(message), + 'signature': encode_hex(Z2_SIGNATURE), + }, + 'output': False, + } + + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... + assert not bls.FastAggregateVerify([], message, NO_SIGNATURE) + yield f'fast_aggregate_verify_na_pubkeys_and_na_signature', { + 'input': { + 'pubkeys': [], + 'message': encode_hex(message), + 'signature': encode_hex(NO_SIGNATURE), + }, + 'output': False, + } + def case05_aggregate_verify(): - pairs = [] + pubkeys = [] + pubkeys_serial = [] + messages = [] + messages_serial = [] sigs = [] for privkey, message in zip(PRIVKEYS, MESSAGES): - sig = bls.G2ProofOfPossession.Sign(privkey, message) - pubkey = bls.G2ProofOfPossession.PrivToPub(privkey) - pairs.append({ - 'pubkey': encode_hex(pubkey), - 'message': encode_hex(message), - }) + sig = bls.Sign(privkey, message) + pubkey = bls.SkToPk(privkey) + pubkeys.append(pubkey) + pubkeys_serial.append(encode_hex(pubkey)) + messages.append(message) + messages_serial.append(encode_hex(message)) sigs.append(sig) - aggregate_signature = bls.G2ProofOfPossession.Aggregate(sigs) + aggregate_signature = bls.Aggregate(sigs) + assert bls.AggregateVerify(pubkeys, messages, aggregate_signature) yield f'aggregate_verify_valid', { 'input': { - 'pairs': pairs, + 'pubkeys': pubkeys_serial, + 'messages': messages_serial, 'signature': encode_hex(aggregate_signature), }, 'output': True, } tampered_signature = aggregate_signature[:4] + b'\xff\xff\xff\xff' + assert not bls.AggregateVerify(pubkey, messages, tampered_signature) yield f'aggregate_verify_tampered_signature', { 'input': { - 'pairs': pairs, + 'pubkeys': pubkeys_serial, + 'messages': messages_serial, 'signature': encode_hex(tampered_signature), }, 'output': False, } + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE + assert not bls.AggregateVerify([], [], Z2_SIGNATURE) + yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { + 'input': { + 'pubkeys': [], + 'messages': [], + 'signature': encode_hex(Z2_SIGNATURE), + }, + 'output': False, + } + + # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... + assert not bls.AggregateVerify([], [], NO_SIGNATURE) + yield f'aggregate_verify_na_pubkeys_and_na_signature', { + 'input': { + 'pubkeys': [], + 'messages': [], + 'signature': encode_hex(NO_SIGNATURE), + }, + 'output': False, + } + def create_provider(handler_name: str, test_case_fn: Callable[[], Iterable[Tuple[str, Dict[str, Any]]]]) -> gen_typing.TestProvider: diff --git a/tests/generators/bls/requirements.txt b/tests/generators/bls/requirements.txt index 24ea127c47..2547052823 100644 --- a/tests/generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,4 +1,4 @@ -py_ecc==2.0.0 +py_ecc==4.0.0 eth-utils==1.6.0 ../../core/gen_helpers ../../../ diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index fd95dcfaad..c90943cabe 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -3,11 +3,9 @@ from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 from eth2spec.test.phase_0.rewards import ( - test_get_source_deltas, - test_get_target_deltas, - test_get_head_deltas, - test_get_inclusion_delay_deltas, - test_get_inactivity_penalty_deltas, + test_basic, + test_leak, + test_random, ) from gen_base import gen_runner, gen_typing from gen_from_tests.gen import generate_from_tests @@ -16,7 +14,7 @@ from eth2spec.test.context import PHASE0 -def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typing.TestProvider: +def create_provider(tests_src, config_name: str) -> gen_typing.TestProvider: def prepare_fn(configs_path: str) -> str: config_util.prepare_config(configs_path, config_name) @@ -27,7 +25,7 @@ def prepare_fn(configs_path: str) -> str: def cases_fn() -> Iterable[gen_typing.TestCase]: return generate_from_tests( runner_name='rewards', - handler_name=handler_name, + handler_name='core', src=tests_src, fork_name=PHASE0, ) @@ -36,15 +34,11 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: if __name__ == "__main__": - gen_runner.run_generator("epoch_processing", [ - create_provider('get_source_deltas', test_get_source_deltas, 'minimal'), - create_provider('get_source_deltas', test_get_source_deltas, 'mainnet'), - create_provider('get_target_deltas', test_get_target_deltas, 'minimal'), - create_provider('get_target_deltas', test_get_target_deltas, 'mainnet'), - create_provider('get_head_deltas', test_get_head_deltas, 'minimal'), - create_provider('get_head_deltas', test_get_head_deltas, 'mainnet'), - create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'minimal'), - create_provider('get_inclusion_delay_deltas', test_get_inclusion_delay_deltas, 'mainnet'), - create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'minimal'), - create_provider('get_inactivity_penalty_deltas', test_get_inactivity_penalty_deltas, 'mainnet'), + gen_runner.run_generator("rewards", [ + create_provider(test_basic, 'minimal'), + create_provider(test_basic, 'mainnet'), + create_provider(test_leak, 'minimal'), + create_provider(test_leak, 'mainnet'), + create_provider(test_random, 'minimal'), + create_provider(test_random, 'mainnet'), ])