From 7230df4a818458f3eb5244c142092d52811c28d0 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 25 Aug 2023 19:15:22 +0300 Subject: [PATCH 01/89] merge from archived repo Signed-off-by: Andrei Sandu --- .gitignore | 16 + polkadot/node/core/approval-voting/Cargo.toml | 6 + .../approval-voting/src/approval_checking.rs | 60 +- .../approval-voting/src/approval_db/mod.rs | 1 + .../approval-voting/src/approval_db/v1/mod.rs | 268 +--- .../src/approval_db/v2/migration_helpers.rs | 241 +++ .../approval-voting/src/approval_db/v2/mod.rs | 394 +++++ .../src/approval_db/v2/tests.rs | 570 +++++++ .../node/core/approval-voting/src/backend.rs | 16 +- .../node/core/approval-voting/src/criteria.rs | 498 ++++-- .../node/core/approval-voting/src/import.rs | 38 +- polkadot/node/core/approval-voting/src/lib.rs | 518 +++++-- polkadot/node/core/approval-voting/src/ops.rs | 2 +- .../approval-voting/src/persisted_entries.rs | 135 +- .../node/core/approval-voting/src/tests.rs | 395 ++++- .../node/core/approval-voting/src/time.rs | 2 +- .../core/dispute-coordinator/src/import.rs | 2 +- .../network/approval-distribution/Cargo.toml | 2 + .../network/approval-distribution/src/lib.rs | 1352 ++++++++++------- .../approval-distribution/src/metrics.rs | 45 +- .../approval-distribution/src/tests.rs | 597 +++++--- .../network/bitfield-distribution/src/lib.rs | 12 +- polkadot/node/network/bridge/src/network.rs | 107 +- polkadot/node/network/bridge/src/rx/mod.rs | 110 +- polkadot/node/network/bridge/src/tx/mod.rs | 91 +- polkadot/node/network/bridge/src/tx/tests.rs | 5 +- polkadot/node/network/protocol/src/lib.rs | 25 +- polkadot/node/primitives/Cargo.toml | 1 + polkadot/node/primitives/src/approval.rs | 655 ++++++-- .../node/service/src/parachains_db/mod.rs | 36 +- .../node/service/src/parachains_db/upgrade.rs | 367 ++++- polkadot/node/service/src/tests.rs | 2 +- polkadot/node/subsystem-types/Cargo.toml | 1 + polkadot/node/subsystem-types/src/messages.rs | 13 +- .../node/subsystem-util/src/reputation.rs | 5 + .../src/node/approval/approval-voting.md | 31 +- .../src/protocol-approval.md | 10 +- .../implementers-guide/src/types/approval.md | 29 + .../implementers-guide/src/types/runtime.md | 2 +- .../scripts/ci/gitlab/pipeline/zombienet.yml | 30 +- .../0005-parachains-max-tranche0.toml | 40 + .../0005-parachains-max-tranche0.zndsl | 27 + .../0002-parachains-upgrade-smoke-test.toml | 2 +- 43 files changed, 4986 insertions(+), 1773 deletions(-) create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs create mode 100644 polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.toml create mode 100644 polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl diff --git a/.gitignore b/.gitignore index e69de29bb2d1..61ef9e91a55e 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,16 @@ +**/target/ +**/*.rs.bk +*.swp +.wasm-binaries +runtime/wasm/target/ +**/._* +.idea +.vscode +polkadot.* +!polkadot.service +.DS_Store +.env + +artifacts +release-artifacts +release.json diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index 5ae99d44d0a5..983189a682c5 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -17,6 +17,7 @@ schnorrkel = "0.9.1" kvdb = "0.13.0" derive_more = "0.99.17" thiserror = "1.0.31" +itertools = "0.10.5" polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } @@ -30,6 +31,9 @@ sp-consensus = { path = "../../../../substrate/primitives/consensus/common", def sp-consensus-slots = { path = "../../../../substrate/primitives/consensus/slots", default-features = false } sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto", default-features = false, features = ["full_crypto"] } sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } +# Needed for migration test helpers +test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } +rand_core = "0.5.1" [dev-dependencies] async-trait = "0.1.57" @@ -43,3 +47,5 @@ polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } assert_matches = "1.4.0" kvdb-memorydb = "0.13.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } +log = "0.4.17" +env_logger = "0.9.0" diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index f345b57029b5..5d24ff164193 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -17,7 +17,7 @@ //! Utilities for checking whether a candidate has been approved under a given block. use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice}; -use polkadot_node_primitives::approval::DelayTranche; +use polkadot_node_primitives::approval::v1::DelayTranche; use polkadot_primitives::ValidatorIndex; use crate::{ @@ -472,9 +472,9 @@ mod tests { } .into(); - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: BitVec::default(), + assigned_validators: BitVec::default(), our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -509,22 +509,22 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 0, assignments: (0..2).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 2, assignments: (5..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -581,22 +581,22 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 0, assignments: (0..4).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (4..6).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 2, assignments: (6..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -647,9 +647,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 5], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 5], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -691,9 +691,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -731,9 +731,9 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; 10], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -776,9 +776,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -843,9 +843,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -934,9 +934,9 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), - assignments: bitvec![u8, BitOrderLsb0; 0; n_validators], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -1041,15 +1041,15 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v1::ApprovalEntry { + let approval_entry = approval_db::v2::ApprovalEntry { tranches: vec![ // Assignments with invalid validator indexes. - approval_db::v1::TrancheEntry { + approval_db::v2::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, ], - assignments: bitvec![u8, BitOrderLsb0; 1; 3], + assigned_validators: bitvec![u8, BitOrderLsb0; 1; 3], our_assignment: None, our_approval_sig: None, backing_group: GroupIndex(0), @@ -1094,12 +1094,12 @@ mod tests { ]; for test_tranche in test_tranches { - let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), backing_group: GroupIndex(0), our_assignment: None, our_approval_sig: None, - assignments: bitvec![u8, BitOrderLsb0; 0; 3], + assigned_validators: bitvec![u8, BitOrderLsb0; 0; 3], approved: false, } .into(); diff --git a/polkadot/node/core/approval-voting/src/approval_db/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/mod.rs index f0ace95613da..20fb6aa82d8d 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/mod.rs @@ -31,3 +31,4 @@ //! time being we share the same DB with the rest of Substrate. pub mod v1; +pub mod v2; diff --git a/polkadot/node/core/approval-voting/src/approval_db/v1/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v1/mod.rs index c31389269d2e..011d0a559c02 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v1/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v1/mod.rs @@ -23,144 +23,15 @@ //! require a db migration (check `node/service/src/parachains_db/upgrade.rs`). use parity_scale_codec::{Decode, Encode}; -use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche}; -use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; -use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_node_primitives::approval::v1::{AssignmentCert, DelayTranche}; use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; +use std::collections::BTreeMap; -use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; -use std::{collections::BTreeMap, sync::Arc}; - -use crate::{ - backend::{Backend, BackendWriteOp}, - persisted_entries, -}; - -const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; - -#[cfg(test)] -pub mod tests; - -/// `DbBackend` is a concrete implementation of the higher-level Backend trait -pub struct DbBackend { - inner: Arc, - config: Config, -} - -impl DbBackend { - /// Create a new [`DbBackend`] with the supplied key-value store and - /// config. - pub fn new(db: Arc, config: Config) -> Self { - DbBackend { inner: db, config } - } -} - -impl Backend for DbBackend { - fn load_block_entry( - &self, - block_hash: &Hash, - ) -> SubsystemResult> { - load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) - } - - fn load_candidate_entry( - &self, - candidate_hash: &CandidateHash, - ) -> SubsystemResult> { - load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) - } - - fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { - load_blocks_at_height(&*self.inner, &self.config, block_height) - } - - fn load_all_blocks(&self) -> SubsystemResult> { - load_all_blocks(&*self.inner, &self.config) - } - - fn load_stored_blocks(&self) -> SubsystemResult> { - load_stored_blocks(&*self.inner, &self.config) - } - - /// Atomically write the list of operations, with later operations taking precedence over prior. - fn write(&mut self, ops: I) -> SubsystemResult<()> - where - I: IntoIterator, - { - let mut tx = DBTransaction::new(); - for op in ops { - match op { - BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { - tx.put_vec( - self.config.col_approval_data, - &STORED_BLOCKS_KEY, - stored_block_range.encode(), - ); - }, - BackendWriteOp::DeleteStoredBlockRange => { - tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); - }, - BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { - tx.put_vec( - self.config.col_approval_data, - &blocks_at_height_key(h), - blocks.encode(), - ); - }, - BackendWriteOp::DeleteBlocksAtHeight(h) => { - tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); - }, - BackendWriteOp::WriteBlockEntry(block_entry) => { - let block_entry: BlockEntry = block_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &block_entry_key(&block_entry.block_hash), - block_entry.encode(), - ); - }, - BackendWriteOp::DeleteBlockEntry(hash) => { - tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); - }, - BackendWriteOp::WriteCandidateEntry(candidate_entry) => { - let candidate_entry: CandidateEntry = candidate_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &candidate_entry_key(&candidate_entry.candidate.hash()), - candidate_entry.encode(), - ); - }, - BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { - tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); - }, - } - } - - self.inner.write(tx).map_err(|e| e.into()) - } -} - -/// A range from earliest..last block number stored within the DB. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); - -// slot_duration * 2 + DelayTranche gives the number of delay tranches since the -// unix epoch. -#[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] -pub struct Tick(u64); - -/// Convenience type definition -pub type Bitfield = BitVec; - -/// The database config. -#[derive(Debug, Clone, Copy)] -pub struct Config { - /// The column family in the database where data is stored. - pub col_approval_data: u32, -} +use super::v2::Bitfield; /// Details pertaining to our assignment on a block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] @@ -171,15 +42,7 @@ pub struct OurAssignment { // Whether the assignment has been triggered already. pub triggered: bool, } - -/// Metadata regarding a specific tranche of assignments for a specific candidate. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct TrancheEntry { - pub tranche: DelayTranche, - // Assigned validators, and the instant we received their assignment, rounded - // to the nearest tick. - pub assignments: Vec<(ValidatorIndex, Tick)>, -} +use super::v2::TrancheEntry; /// Metadata regarding approval of a particular candidate within the context of some /// particular block. @@ -226,126 +89,3 @@ pub struct BlockEntry { pub approved_bitfield: Bitfield, pub children: Vec, } - -impl From for Tick { - fn from(tick: crate::Tick) -> Tick { - Tick(tick) - } -} - -impl From for crate::Tick { - fn from(tick: Tick) -> crate::Tick { - tick.0 - } -} - -/// Errors while accessing things from the DB. -#[derive(Debug, derive_more::From, derive_more::Display)] -pub enum Error { - Io(std::io::Error), - InvalidDecoding(parity_scale_codec::Error), -} - -impl std::error::Error for Error {} - -/// Result alias for DB errors. -pub type Result = std::result::Result; - -pub(crate) fn load_decode( - store: &dyn Database, - col_approval_data: u32, - key: &[u8], -) -> Result> { - match store.get(col_approval_data, key)? { - None => Ok(None), - Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), - } -} - -/// The key a given block entry is stored under. -pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { - const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(block_hash.as_ref()); - - key -} - -/// The key a given candidate entry is stored under. -pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { - const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); - - key -} - -/// The key a set of block hashes corresponding to a block number is stored under. -pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { - const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; - - let mut key = [0u8; 12 + 4]; - key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); - block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); - - key -} - -/// Return all blocks which have entries in the DB, ascending, by height. -pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { - let mut hashes = Vec::new(); - if let Some(stored_blocks) = load_stored_blocks(store, config)? { - for height in stored_blocks.0..stored_blocks.1 { - let blocks = load_blocks_at_height(store, config, &height)?; - hashes.extend(blocks); - } - } - - Ok(hashes) -} - -/// Load the stored-blocks key from the state. -pub fn load_stored_blocks( - store: &dyn Database, - config: &Config, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a blocks-at-height entry for a given block number. -pub fn load_blocks_at_height( - store: &dyn Database, - config: &Config, - block_number: &BlockNumber, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) - .map(|x| x.unwrap_or_default()) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a block entry from the aux store. -pub fn load_block_entry( - store: &dyn Database, - config: &Config, - block_hash: &Hash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a candidate entry from the aux store. -pub fn load_candidate_entry( - store: &dyn Database, - config: &Config, - candidate_hash: &CandidateHash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs new file mode 100644 index 000000000000..3d1f28ee1937 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -0,0 +1,241 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Approval DB migration helpers. +use super::{StoredBlockRange, *}; +use crate::backend::Backend; +use polkadot_node_primitives::approval::v1::{ + AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, +}; +use polkadot_node_subsystem_util::database::Database; +use std::{collections::HashSet, sync::Arc}; + +use ::test_helpers::dummy_candidate_receipt; + +fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} + +fn make_block_entry_v1( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> crate::approval_db::v1::BlockEntry { + crate::approval_db::v1::BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + } +} + +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +/// Migrates `OurAssignment`, `CandidateEntry` and `ApprovalEntry` to version 2. +/// Returns on any error. +/// Must only be used in parachains DB migration code - `polkadot-service` crate. +pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { + let mut backend = crate::DbBackend::new(db, config); + let all_blocks = backend + .load_all_blocks() + .map_err(|e| Error::InternalError(e))? + .iter() + .filter_map(|block_hash| { + backend + .load_block_entry_v1(block_hash) + .map_err(|e| Error::InternalError(e)) + .ok()? + }) + .collect::>(); + + gum::info!( + target: crate::LOG_TARGET, + "Migrating candidate entries on top of {} blocks", + all_blocks.len() + ); + + let mut overlay = crate::OverlayedBackend::new(&backend); + let mut counter = 0; + // Get all candidate entries, approval entries and convert each of them. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend + .load_candidate_entry_v1(&candidate_hash) + .map_err(|e| Error::InternalError(e))? + { + // Write the updated representation. + overlay.write_candidate_entry(candidate_entry); + counter += 1; + } + } + overlay.write_block_entry(block); + } + + gum::info!(target: crate::LOG_TARGET, "Migrated {} entries", counter); + + // Commit all changes to DB. + let write_ops = overlay.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(()) +} + +// Checks if the migration doesn't leave the DB in an unsane state. +// This function is to be used in tests. +pub fn v1_to_v2_sanity_check( + db: Arc, + config: Config, + expected_candidates: HashSet, +) -> Result<()> { + let backend = crate::DbBackend::new(db, config); + + let all_blocks = backend + .load_all_blocks() + .unwrap() + .iter() + .map(|block_hash| backend.load_block_entry(block_hash).unwrap().unwrap()) + .collect::>(); + + let mut candidates = HashSet::new(); + + // Iterate all blocks and approval entries. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend.load_candidate_entry(&candidate_hash).unwrap() { + candidates.insert(candidate_entry.candidate.hash()); + } + } + } + + assert_eq!(candidates, expected_candidates); + + Ok(()) +} + +// Fills the db with dummy data in v1 scheme. +pub fn v1_to_v2_fill_test_data( + db: Arc, + config: Config, +) -> Result> { + let mut backend = crate::DbBackend::new(db.clone(), config); + let mut overlay_db = crate::OverlayedBackend::new(&backend); + let mut expected_candidates = HashSet::new(); + + const RELAY_BLOCK_COUNT: u32 = 10; + + let range = StoredBlockRange(1, 11); + overlay_db.write_stored_block_range(range.clone()); + + for relay_number in 1..=RELAY_BLOCK_COUNT { + let relay_hash = Hash::repeat_byte(relay_number as u8); + let assignment_core_index = CoreIndex(relay_number); + let candidate = dummy_candidate_receipt(relay_hash); + let candidate_hash = candidate.hash(); + + let at_height = vec![relay_hash]; + + let block_entry = make_block_entry_v1( + relay_hash, + Default::default(), + relay_number, + vec![(assignment_core_index, candidate_hash)], + ); + + let dummy_assignment = crate::approval_db::v1::OurAssignment { + cert: dummy_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }).into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + }; + + let candidate_entry = crate::approval_db::v1::CandidateEntry { + candidate, + session: 123, + block_assignments: vec![( + relay_hash, + crate::approval_db::v1::ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: Some(dummy_assignment), + our_approval_sig: None, + assignments: Default::default(), + approved: false, + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + overlay_db.write_blocks_at_height(relay_number, at_height.clone()); + expected_candidates.insert(candidate_entry.candidate.hash()); + + db.write(write_candidate_entry_v1(candidate_entry, config)).unwrap(); + db.write(write_block_entry_v1(block_entry, config)).unwrap(); + } + + let write_ops = overlay_db.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(expected_candidates) +} + +// Low level DB helper to write a candidate entry in v1 scheme. +fn write_candidate_entry_v1( + candidate_entry: crate::approval_db::v1::CandidateEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + tx +} + +// Low level DB helper to write a block entry in v1 scheme. +fn write_block_entry_v1( + block_entry: crate::approval_db::v1::BlockEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + tx +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs new file mode 100644 index 000000000000..66df6ee8f653 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -0,0 +1,394 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Version 2 of the DB schema. + +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_primitives::approval::{v1::DelayTranche, v2::AssignmentCertV2}; +use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; +use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, + ValidatorIndex, ValidatorSignature, +}; + +use sp_consensus_slots::Slot; + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; +use std::{collections::BTreeMap, sync::Arc}; + +use crate::{ + backend::{Backend, BackendWriteOp, V1ReadBackend}, + persisted_entries, +}; + +const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; + +pub mod migration_helpers; +#[cfg(test)] +pub mod tests; + +/// `DbBackend` is a concrete implementation of the higher-level Backend trait +pub struct DbBackend { + inner: Arc, + config: Config, +} + +impl DbBackend { + /// Create a new [`DbBackend`] with the supplied key-value store and + /// config. + pub fn new(db: Arc, config: Config) -> Self { + DbBackend { inner: db, config } + } +} + +impl V1ReadBackend for DbBackend { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) + .map(|e| e.map(Into::into)) + } + + fn load_block_entry_v1( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry_v1(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } +} + +impl Backend for DbBackend { + fn load_block_entry( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } + + fn load_candidate_entry( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) + } + + fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { + load_blocks_at_height(&*self.inner, &self.config, block_height) + } + + fn load_all_blocks(&self) -> SubsystemResult> { + load_all_blocks(&*self.inner, &self.config) + } + + fn load_stored_blocks(&self) -> SubsystemResult> { + load_stored_blocks(&*self.inner, &self.config) + } + + /// Atomically write the list of operations, with later operations taking precedence over prior. + fn write(&mut self, ops: I) -> SubsystemResult<()> + where + I: IntoIterator, + { + let mut tx = DBTransaction::new(); + for op in ops { + match op { + BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { + tx.put_vec( + self.config.col_approval_data, + &STORED_BLOCKS_KEY, + stored_block_range.encode(), + ); + }, + BackendWriteOp::DeleteStoredBlockRange => { + tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); + }, + BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { + tx.put_vec( + self.config.col_approval_data, + &blocks_at_height_key(h), + blocks.encode(), + ); + }, + BackendWriteOp::DeleteBlocksAtHeight(h) => { + tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); + }, + BackendWriteOp::WriteBlockEntry(block_entry) => { + let block_entry: BlockEntry = block_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + }, + BackendWriteOp::DeleteBlockEntry(hash) => { + tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); + }, + BackendWriteOp::WriteCandidateEntry(candidate_entry) => { + let candidate_entry: CandidateEntry = candidate_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + }, + BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { + tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); + }, + } + } + + self.inner.write(tx).map_err(|e| e.into()) + } +} + +/// A range from earliest..last block number stored within the DB. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); + +// slot_duration * 2 + DelayTranche gives the number of delay tranches since the +// unix epoch. +#[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] +pub struct Tick(u64); + +/// Convenience type definition +pub type Bitfield = BitVec; + +/// The database config. +#[derive(Debug, Clone, Copy)] +pub struct Config { + /// The column family in the database where data is stored. + pub col_approval_data: u32, +} + +/// Details pertaining to our assignment on a block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct OurAssignment { + /// Our assignment certificate. + pub cert: AssignmentCertV2, + /// The tranche for which the assignment refers to. + pub tranche: DelayTranche, + /// Our validator index for the session in which the candidates were included. + pub validator_index: ValidatorIndex, + /// Whether the assignment has been triggered already. + pub triggered: bool, +} + +/// Metadata regarding a specific tranche of assignments for a specific candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct TrancheEntry { + pub tranche: DelayTranche, + // Assigned validators, and the instant we received their assignment, rounded + // to the nearest tick. + pub assignments: Vec<(ValidatorIndex, Tick)>, +} + +/// Metadata regarding approval of a particular candidate within the context of some +/// particular block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct ApprovalEntry { + pub tranches: Vec, + pub backing_group: GroupIndex, + pub our_assignment: Option, + pub our_approval_sig: Option, + // `n_validators` bits. + pub assigned_validators: Bitfield, + pub approved: bool, +} + +/// Metadata regarding approval of a particular candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct CandidateEntry { + pub candidate: CandidateReceipt, + pub session: SessionIndex, + // Assignments are based on blocks, so we need to track assignments separately + // based on the block we are looking at. + pub block_assignments: BTreeMap, + pub approvals: Bitfield, +} + +/// Metadata regarding approval of a particular block, by way of approval of the +/// candidates contained within it. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct BlockEntry { + pub block_hash: Hash, + pub block_number: BlockNumber, + pub parent_hash: Hash, + pub session: SessionIndex, + pub slot: Slot, + /// Random bytes derived from the VRF submitted within the block by the block + /// author as a credential and used as input to approval assignment criteria. + pub relay_vrf_story: [u8; 32], + // The candidates included as-of this block and the index of the core they are + // leaving. Sorted ascending by core index. + pub candidates: Vec<(CoreIndex, CandidateHash)>, + // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. + // The i'th bit is `true` iff the candidate has been approved in the context of this + // block. The block can be considered approved if the bitfield has all bits set to `true`. + pub approved_bitfield: Bitfield, + pub children: Vec, + // Assignments we already distributed. A 1 bit means the candidate index for which + // we already have sent out an assignment. We need this to avoid distributing + // multiple core assignments more than once. + pub distributed_assignments: Bitfield, +} + +impl From for Tick { + fn from(tick: crate::Tick) -> Tick { + Tick(tick) + } +} + +impl From for crate::Tick { + fn from(tick: Tick) -> crate::Tick { + tick.0 + } +} + +/// Errors while accessing things from the DB. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum Error { + Io(std::io::Error), + InvalidDecoding(parity_scale_codec::Error), + InternalError(SubsystemError), +} + +impl std::error::Error for Error {} + +/// Result alias for DB errors. +pub type Result = std::result::Result; + +pub(crate) fn load_decode( + store: &dyn Database, + col_approval_data: u32, + key: &[u8], +) -> Result> { + match store.get(col_approval_data, key)? { + None => Ok(None), + Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), + } +} + +/// The key a given block entry is stored under. +pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { + const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(block_hash.as_ref()); + + key +} + +/// The key a given candidate entry is stored under. +pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { + const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); + + key +} + +/// The key a set of block hashes corresponding to a block number is stored under. +pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { + const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; + + let mut key = [0u8; 12 + 4]; + key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); + block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); + + key +} + +/// Return all blocks which have entries in the DB, ascending, by height. +pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { + let mut hashes = Vec::new(); + if let Some(stored_blocks) = load_stored_blocks(store, config)? { + for height in stored_blocks.0..stored_blocks.1 { + let blocks = load_blocks_at_height(store, config, &height)?; + hashes.extend(blocks); + } + } + + Ok(hashes) +} + +/// Load the stored-blocks key from the state. +pub fn load_stored_blocks( + store: &dyn Database, + config: &Config, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a blocks-at-height entry for a given block number. +pub fn load_blocks_at_height( + store: &dyn Database, + config: &Config, + block_number: &BlockNumber, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) + .map(|x| x.unwrap_or_default()) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store. +pub fn load_block_entry( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in current version format. +pub fn load_candidate_entry( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in v1 format. +pub fn load_candidate_entry_v1( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store in v1 format. +pub fn load_block_entry_v1( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs new file mode 100644 index 000000000000..50a5a924ca8d --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -0,0 +1,570 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Tests for the aux-schema of approval voting. + +use super::{DbBackend, StoredBlockRange, *}; +use crate::{ + backend::{Backend, OverlayedBackend}, + ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo}, +}; +use polkadot_node_subsystem_util::database::Database; +use polkadot_primitives::Id as ParaId; +use std::{collections::HashMap, sync::Arc}; + +use ::test_helpers::{dummy_candidate_receipt, dummy_candidate_receipt_bad_sig, dummy_hash}; + +const DATA_COL: u32 = 0; + +const NUM_COLUMNS: u32 = 1; + +const TEST_CONFIG: Config = Config { col_approval_data: DATA_COL }; + +fn make_db() -> (DbBackend, Arc) { + let db = kvdb_memorydb::create(NUM_COLUMNS); + let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); + let db_writer: Arc = Arc::new(db); + (DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer) +} + +fn make_block_entry( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> BlockEntry { + BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + distributed_assignments: Default::default(), + } +} + +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { + let mut c = dummy_candidate_receipt(dummy_hash()); + + c.descriptor.para_id = para_id; + c.descriptor.relay_parent = relay_parent; + + c +} + +#[test] +fn read_write() { + let (mut db, store) = make_db(); + + let hash_a = Hash::repeat_byte(1); + let hash_b = Hash::repeat_byte(2); + let candidate_hash = dummy_candidate_receipt_bad_sig(dummy_hash(), None).hash(); + + let range = StoredBlockRange(10, 20); + let at_height = vec![hash_a, hash_b]; + + let block_entry = + make_block_entry(hash_a, Default::default(), 1, vec![(CoreIndex(0), candidate_hash)]); + + let candidate_entry = CandidateEntry { + candidate: dummy_candidate_receipt_bad_sig(dummy_hash(), None), + session: 5, + block_assignments: vec![( + hash_a, + ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: None, + our_approval_sig: None, + assigned_validators: Default::default(), + approved: false, + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(range.clone()); + overlay_db.write_blocks_at_height(1, at_height.clone()); + overlay_db.write_block_entry(block_entry.clone().into()); + overlay_db.write_candidate_entry(candidate_entry.clone().into()); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range)); + assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(), + Some(block_entry.into()) + ); + assert_eq!( + load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(), + Some(candidate_entry.into()), + ); + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.delete_blocks_at_height(1); + overlay_db.delete_block_entry(&hash_a); + overlay_db.delete_candidate_entry(&candidate_hash); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none()); + assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash) + .unwrap() + .is_none()); +} + +#[test] +fn add_block_entry_works() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + + let candidate_receipt_a = make_candidate(ParaId::from(1_u32), parent_hash); + let candidate_receipt_b = make_candidate(ParaId::from(2_u32), parent_hash); + + let candidate_hash_a = candidate_receipt_a.hash(); + let candidate_hash_b = candidate_receipt_b.hash(); + + let block_number = 10; + + let block_entry_a = make_block_entry( + block_hash_a, + parent_hash, + block_number, + vec![(CoreIndex(0), candidate_hash_a)], + ); + + let block_entry_b = make_block_entry( + block_hash_b, + parent_hash, + block_number, + vec![(CoreIndex(0), candidate_hash_a), (CoreIndex(1), candidate_hash_b)], + ); + + let n_validators = 10; + + let mut new_candidate_info = HashMap::new(); + new_candidate_info + .insert(candidate_hash_a, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(0), None)); + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |h| { + new_candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + new_candidate_info + .insert(candidate_hash_b, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(1), None)); + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |h| { + new_candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + Some(block_entry_a.into()) + ); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + Some(block_entry_b.into()) + ); + + let candidate_entry_a = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_a) + .unwrap() + .unwrap(); + assert_eq!( + candidate_entry_a.block_assignments.keys().collect::>(), + vec![&block_hash_a, &block_hash_b] + ); + + let candidate_entry_b = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_b) + .unwrap() + .unwrap(); + assert_eq!(candidate_entry_b.block_assignments.keys().collect::>(), vec![&block_hash_b]); +} + +#[test] +fn add_block_entry_adds_child() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + + let mut block_entry_a = make_block_entry(block_hash_a, parent_hash, 1, Vec::new()); + + let block_entry_b = make_block_entry(block_hash_b, block_hash_a, 2, Vec::new()); + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap(); + + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap(); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + block_entry_a.children.push(block_hash_b); + + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + Some(block_entry_a.into()) + ); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + Some(block_entry_b.into()) + ); +} + +#[test] +fn canonicalize_works() { + let (mut db, store) = make_db(); + + // -> B1 -> C1 -> D1 + // A -> B2 -> C2 -> D2 + // + // We'll canonicalize C1. Everytning except D1 should disappear. + // + // Candidates: + // Cand1 in B2 + // Cand2 in C2 + // Cand3 in C2 and D1 + // Cand4 in D1 + // Cand5 in D2 + // Only Cand3 and Cand4 should remain after canonicalize. + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(StoredBlockRange(1, 5)); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let genesis = Hash::repeat_byte(0); + + let block_hash_a = Hash::repeat_byte(1); + let block_hash_b1 = Hash::repeat_byte(2); + let block_hash_b2 = Hash::repeat_byte(3); + let block_hash_c1 = Hash::repeat_byte(4); + let block_hash_c2 = Hash::repeat_byte(5); + let block_hash_d1 = Hash::repeat_byte(6); + let block_hash_d2 = Hash::repeat_byte(7); + + let candidate_receipt_genesis = make_candidate(ParaId::from(1_u32), genesis); + let candidate_receipt_a = make_candidate(ParaId::from(2_u32), block_hash_a); + let candidate_receipt_b = make_candidate(ParaId::from(3_u32), block_hash_a); + let candidate_receipt_b1 = make_candidate(ParaId::from(4_u32), block_hash_b1); + let candidate_receipt_c1 = make_candidate(ParaId::from(5_u32), block_hash_c1); + + let cand_hash_1 = candidate_receipt_genesis.hash(); + let cand_hash_2 = candidate_receipt_a.hash(); + let cand_hash_3 = candidate_receipt_b.hash(); + let cand_hash_4 = candidate_receipt_b1.hash(); + let cand_hash_5 = candidate_receipt_c1.hash(); + + let block_entry_a = make_block_entry(block_hash_a, genesis, 1, Vec::new()); + let block_entry_b1 = make_block_entry(block_hash_b1, block_hash_a, 2, Vec::new()); + let block_entry_b2 = + make_block_entry(block_hash_b2, block_hash_a, 2, vec![(CoreIndex(0), cand_hash_1)]); + let block_entry_c1 = make_block_entry(block_hash_c1, block_hash_b1, 3, Vec::new()); + let block_entry_c2 = make_block_entry( + block_hash_c2, + block_hash_b2, + 3, + vec![(CoreIndex(0), cand_hash_2), (CoreIndex(1), cand_hash_3)], + ); + let block_entry_d1 = make_block_entry( + block_hash_d1, + block_hash_c1, + 4, + vec![(CoreIndex(0), cand_hash_3), (CoreIndex(1), cand_hash_4)], + ); + let block_entry_d2 = + make_block_entry(block_hash_d2, block_hash_c2, 4, vec![(CoreIndex(0), cand_hash_5)]); + + let candidate_info = { + let mut candidate_info = HashMap::new(); + candidate_info.insert( + cand_hash_1, + NewCandidateInfo::new(candidate_receipt_genesis, GroupIndex(1), None), + ); + + candidate_info + .insert(cand_hash_2, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(2), None)); + + candidate_info + .insert(cand_hash_3, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(3), None)); + + candidate_info + .insert(cand_hash_4, NewCandidateInfo::new(candidate_receipt_b1, GroupIndex(4), None)); + + candidate_info + .insert(cand_hash_5, NewCandidateInfo::new(candidate_receipt_c1, GroupIndex(5), None)); + + candidate_info + }; + + // now insert all the blocks. + let blocks = vec![ + block_entry_a.clone(), + block_entry_b1.clone(), + block_entry_b2.clone(), + block_entry_c1.clone(), + block_entry_c2.clone(), + block_entry_d1.clone(), + block_entry_d2.clone(), + ]; + + let mut overlay_db = OverlayedBackend::new(&db); + for block_entry in blocks { + add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| { + candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + } + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let check_candidates_in_store = |expected: Vec<(CandidateHash, Option>)>| { + for (c_hash, in_blocks) in expected { + let (entry, in_blocks) = match in_blocks { + None => { + assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash) + .unwrap() + .is_none()); + continue + }, + Some(i) => ( + load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().unwrap(), + i, + ), + }; + + assert_eq!(entry.block_assignments.len(), in_blocks.len()); + + for x in in_blocks { + assert!(entry.block_assignments.contains_key(&x)); + } + } + }; + + let check_blocks_in_store = |expected: Vec<(Hash, Option>)>| { + for (hash, with_candidates) in expected { + let (entry, with_candidates) = match with_candidates { + None => { + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash) + .unwrap() + .is_none()); + continue + }, + Some(i) => + (load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i), + }; + + assert_eq!(entry.candidates.len(), with_candidates.len()); + + for x in with_candidates { + assert!(entry.candidates.iter().any(|(_, c)| c == &x)); + } + } + }; + + check_candidates_in_store(vec![ + (cand_hash_1, Some(vec![block_hash_b2])), + (cand_hash_2, Some(vec![block_hash_c2])), + (cand_hash_3, Some(vec![block_hash_c2, block_hash_d1])), + (cand_hash_4, Some(vec![block_hash_d1])), + (cand_hash_5, Some(vec![block_hash_d2])), + ]); + + check_blocks_in_store(vec![ + (block_hash_a, Some(vec![])), + (block_hash_b1, Some(vec![])), + (block_hash_b2, Some(vec![cand_hash_1])), + (block_hash_c1, Some(vec![])), + (block_hash_c2, Some(vec![cand_hash_2, cand_hash_3])), + (block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])), + (block_hash_d2, Some(vec![cand_hash_5])), + ]); + + let mut overlay_db = OverlayedBackend::new(&db); + canonicalize(&mut overlay_db, 3, block_hash_c1).unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(), + StoredBlockRange(4, 5) + ); + + check_candidates_in_store(vec![ + (cand_hash_1, None), + (cand_hash_2, None), + (cand_hash_3, Some(vec![block_hash_d1])), + (cand_hash_4, Some(vec![block_hash_d1])), + (cand_hash_5, None), + ]); + + check_blocks_in_store(vec![ + (block_hash_a, None), + (block_hash_b1, None), + (block_hash_b2, None), + (block_hash_c1, None), + (block_hash_c2, None), + (block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])), + (block_hash_d2, None), + ]); +} + +#[test] +fn force_approve_works() { + let (mut db, store) = make_db(); + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(StoredBlockRange(1, 4)); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let single_candidate_vec = vec![(CoreIndex(0), candidate_hash)]; + let candidate_info = { + let mut candidate_info = HashMap::new(); + candidate_info.insert( + candidate_hash, + NewCandidateInfo::new( + make_candidate(ParaId::from(1_u32), Default::default()), + GroupIndex(1), + None, + ), + ); + + candidate_info + }; + + let block_hash_a = Hash::repeat_byte(1); // 1 + let block_hash_b = Hash::repeat_byte(2); + let block_hash_c = Hash::repeat_byte(3); + let block_hash_d = Hash::repeat_byte(4); // 4 + + let block_entry_a = + make_block_entry(block_hash_a, Default::default(), 1, single_candidate_vec.clone()); + let block_entry_b = + make_block_entry(block_hash_b, block_hash_a, 2, single_candidate_vec.clone()); + let block_entry_c = + make_block_entry(block_hash_c, block_hash_b, 3, single_candidate_vec.clone()); + let block_entry_d = + make_block_entry(block_hash_d, block_hash_c, 4, single_candidate_vec.clone()); + + let blocks = vec![ + block_entry_a.clone(), + block_entry_b.clone(), + block_entry_c.clone(), + block_entry_d.clone(), + ]; + + let mut overlay_db = OverlayedBackend::new(&db); + for block_entry in blocks { + add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| { + candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + } + let approved_hashes = force_approve(&mut overlay_db, block_hash_d, 2).unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a,) + .unwrap() + .unwrap() + .approved_bitfield + .all()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b,) + .unwrap() + .unwrap() + .approved_bitfield + .all()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_c,) + .unwrap() + .unwrap() + .approved_bitfield + .not_any()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_d,) + .unwrap() + .unwrap() + .approved_bitfield + .not_any()); + assert_eq!(approved_hashes, vec![block_hash_b, block_hash_a]); +} + +#[test] +fn load_all_blocks_works() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + let block_hash_c = Hash::repeat_byte(42); + + let block_number = 10; + + let block_entry_a = make_block_entry(block_hash_a, parent_hash, block_number, vec![]); + + let block_entry_b = make_block_entry(block_hash_b, parent_hash, block_number, vec![]); + + let block_entry_c = make_block_entry(block_hash_c, block_hash_a, block_number + 1, vec![]); + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap(); + + // add C before B to test sorting. + add_block_entry(&mut overlay_db, block_entry_c.clone().into(), n_validators, |_| None).unwrap(); + + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap(); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_all_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), + vec![block_hash_a, block_hash_b, block_hash_c], + ) +} diff --git a/polkadot/node/core/approval-voting/src/backend.rs b/polkadot/node/core/approval-voting/src/backend.rs index 87d67c52c467..374e7a826d19 100644 --- a/polkadot/node/core/approval-voting/src/backend.rs +++ b/polkadot/node/core/approval-voting/src/backend.rs @@ -27,7 +27,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, Hash}; use std::collections::HashMap; use super::{ - approval_db::v1::StoredBlockRange, + approval_db::v2::StoredBlockRange, persisted_entries::{BlockEntry, CandidateEntry}, }; @@ -44,6 +44,7 @@ pub enum BackendWriteOp { } /// An abstraction over backend storage for the logic of this subsystem. +/// Implementation must always target latest storage version. pub trait Backend { /// Load a block entry from the DB. fn load_block_entry(&self, hash: &Hash) -> SubsystemResult>; @@ -52,6 +53,7 @@ pub trait Backend { &self, candidate_hash: &CandidateHash, ) -> SubsystemResult>; + /// Load all blocks at a specific height. fn load_blocks_at_height(&self, height: &BlockNumber) -> SubsystemResult>; /// Load all block from the DB. @@ -64,6 +66,18 @@ pub trait Backend { I: IntoIterator; } +/// A read only backed to enable db migration from version 1 of DB. +pub trait V1ReadBackend: Backend { + /// Load a candidate entry from the DB with scheme version 1. + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult>; + + /// Load a block entry from the DB with scheme version 1. + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult>; +} + // Status of block range in the `OverlayedBackend`. #[derive(PartialEq)] enum BlockRangeStatus { diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 0e1d18198c21..1f751e2bf140 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -18,7 +18,9 @@ use parity_scale_codec::{Decode, Encode}; use polkadot_node_primitives::approval::{ - self as approval_types, AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory, + self as approval_types, + v1::{AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory}, + v2::{AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, VrfOutput, VrfProof, VrfSignature}, }; use polkadot_primitives::{ AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, @@ -30,6 +32,7 @@ use sp_application_crypto::ByteArray; use merlin::Transcript; use schnorrkel::vrf::VRFInOut; +use itertools::Itertools; use std::collections::{hash_map::Entry, HashMap}; use super::LOG_TARGET; @@ -37,7 +40,7 @@ use super::LOG_TARGET; /// Details pertaining to our assignment on a block. #[derive(Debug, Clone, Encode, Decode, PartialEq)] pub struct OurAssignment { - cert: AssignmentCert, + cert: AssignmentCertV2, tranche: DelayTranche, validator_index: ValidatorIndex, // Whether the assignment has been triggered already. @@ -45,7 +48,7 @@ pub struct OurAssignment { } impl OurAssignment { - pub(crate) fn cert(&self) -> &AssignmentCert { + pub(crate) fn cert(&self) -> &AssignmentCertV2 { &self.cert } @@ -66,8 +69,8 @@ impl OurAssignment { } } -impl From for OurAssignment { - fn from(entry: crate::approval_db::v1::OurAssignment) -> Self { +impl From for OurAssignment { + fn from(entry: crate::approval_db::v2::OurAssignment) -> Self { OurAssignment { cert: entry.cert, tranche: entry.tranche, @@ -77,7 +80,7 @@ impl From for OurAssignment { } } -impl From for crate::approval_db::v1::OurAssignment { +impl From for crate::approval_db::v2::OurAssignment { fn from(entry: OurAssignment) -> Self { Self { cert: entry.cert, @@ -88,17 +91,97 @@ impl From for crate::approval_db::v1::OurAssignment { } } -fn relay_vrf_modulo_transcript(relay_vrf_story: RelayVRFStory, sample: u32) -> Transcript { - // combine the relay VRF story with a sample number. - let mut t = Transcript::new(approval_types::RELAY_VRF_MODULO_CONTEXT); - t.append_message(b"RC-VRF", &relay_vrf_story.0); - sample.using_encoded(|s| t.append_message(b"sample", s)); +// Combines the relay VRF story with a sample number if any. +fn relay_vrf_modulo_transcript_inner( + mut transcript: Transcript, + relay_vrf_story: RelayVRFStory, + sample: Option, +) -> Transcript { + transcript.append_message(b"RC-VRF", &relay_vrf_story.0); - t + if let Some(sample) = sample { + sample.using_encoded(|s| transcript.append_message(b"sample", s)); + } + + transcript +} + +fn relay_vrf_modulo_transcript_v1(relay_vrf_story: RelayVRFStory, sample: u32) -> Transcript { + relay_vrf_modulo_transcript_inner( + Transcript::new(approval_types::v1::RELAY_VRF_MODULO_CONTEXT), + relay_vrf_story, + Some(sample), + ) +} + +fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript { + relay_vrf_modulo_transcript_inner( + Transcript::new(approval_types::v2::RELAY_VRF_MODULO_CONTEXT), + relay_vrf_story, + None, + ) +} + +/// A hard upper bound on num_cores * target_checkers / num_validators +const MAX_MODULO_SAMPLES: usize = 40; + +use std::convert::AsMut; + +fn clone_into_array(slice: &[T]) -> A +where + A: Default + AsMut<[T]>, + T: Clone, +{ + let mut a = A::default(); + >::as_mut(&mut a).clone_from_slice(slice); + a +} + +struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); + +impl Default for BigArray { + fn default() -> Self { + BigArray([0u8; MAX_MODULO_SAMPLES * 4]) + } +} + +impl AsMut<[u8]> for BigArray { + fn as_mut(&mut self) -> &mut [u8] { + self.0.as_mut() + } +} + +/// Takes the VRF output as input and returns a Vec of cores the validator is assigned +/// to as a tranche0 checker. +fn relay_vrf_modulo_cores( + vrf_in_out: &VRFInOut, + // Configuration - `relay_vrf_modulo_samples`. + num_samples: u32, + // Configuration - `n_cores`. + max_cores: u32, +) -> Vec { + if num_samples as usize > MAX_MODULO_SAMPLES { + gum::warn!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "`num_samples` is greater than `MAX_MODULO_SAMPLES`", + ); + } + + vrf_in_out + .make_bytes::(approval_types::v2::CORE_RANDOMNESS_CONTEXT) + .0 + .chunks_exact(4) + .take(num_samples as usize) + .map(move |sample| CoreIndex(u32::from_le_bytes(clone_into_array(&sample)) % max_cores)) + .unique() + .collect::>() } fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { - let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::CORE_RANDOMNESS_CONTEXT); + let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::v1::CORE_RANDOMNESS_CONTEXT); // interpret as little-endian u32. let random_core = u32::from_le_bytes(bytes) % n_cores; @@ -106,7 +189,7 @@ fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { } fn relay_vrf_delay_transcript(relay_vrf_story: RelayVRFStory, core_index: CoreIndex) -> Transcript { - let mut t = Transcript::new(approval_types::RELAY_VRF_DELAY_CONTEXT); + let mut t = Transcript::new(approval_types::v1::RELAY_VRF_DELAY_CONTEXT); t.append_message(b"RC-VRF", &relay_vrf_story.0); core_index.0.using_encoded(|s| t.append_message(b"core", s)); t @@ -117,7 +200,7 @@ fn relay_vrf_delay_tranche( num_delay_tranches: u32, zeroth_delay_tranche_width: u32, ) -> DelayTranche { - let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::TRANCHE_RANDOMNESS_CONTEXT); + let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::v1::TRANCHE_RANDOMNESS_CONTEXT); // interpret as little-endian u32 and reduce by the number of tranches. let wide_tranche = @@ -128,13 +211,13 @@ fn relay_vrf_delay_tranche( } fn assigned_core_transcript(core_index: CoreIndex) -> Transcript { - let mut t = Transcript::new(approval_types::ASSIGNED_CORE_CONTEXT); + let mut t = Transcript::new(approval_types::v1::ASSIGNED_CORE_CONTEXT); core_index.0.using_encoded(|s| t.append_message(b"core", s)); t } /// Information about the world assignments are being produced in. -#[derive(Clone)] +#[derive(Clone, Debug)] pub(crate) struct Config { /// The assignment public keys for validators. assignment_keys: Vec, @@ -175,12 +258,13 @@ pub(crate) trait AssignmentCriteria { fn check_assignment_cert( &self, - claimed_core_index: CoreIndex, + claimed_core_bitfield: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + // Backing groups for each "leaving core". + backing_groups: Vec, ) -> Result; } @@ -194,25 +278,25 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) } fn check_assignment_cert( &self, - claimed_core_index: CoreIndex, + claimed_core_bitfield: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + backing_groups: Vec, ) -> Result { check_assignment_cert( - claimed_core_index, + claimed_core_bitfield, validator_index, config, relay_vrf_story, assignment, - backing_group, + backing_groups, ) } } @@ -233,6 +317,7 @@ pub(crate) fn compute_assignments( relay_vrf_story: RelayVRFStory, config: &Config, leaving_cores: impl IntoIterator + Clone, + enable_v2_assignments: bool, ) -> HashMap { if config.n_cores == 0 || config.assignment_keys.is_empty() || @@ -291,14 +376,25 @@ pub(crate) fn compute_assignments( let mut assignments = HashMap::new(); // First run `RelayVRFModulo` for each sample. - compute_relay_vrf_modulo_assignments( - &assignments_key, - index, - config, - relay_vrf_story.clone(), - leaving_cores.iter().cloned(), - &mut assignments, - ); + if enable_v2_assignments { + compute_relay_vrf_modulo_assignments_v2( + &assignments_key, + index, + config, + relay_vrf_story.clone(), + leaving_cores.clone(), + &mut assignments, + ); + } else { + compute_relay_vrf_modulo_assignments_v1( + &assignments_key, + index, + config, + relay_vrf_story.clone(), + leaving_cores.clone(), + &mut assignments, + ); + } // Then run `RelayVRFDelay` once for the whole block. compute_relay_vrf_delay_assignments( @@ -313,7 +409,7 @@ pub(crate) fn compute_assignments( assignments } -fn compute_relay_vrf_modulo_assignments( +fn compute_relay_vrf_modulo_assignments_v1( assignments_key: &schnorrkel::Keypair, validator_index: ValidatorIndex, config: &Config, @@ -329,7 +425,7 @@ fn compute_relay_vrf_modulo_assignments( // into closure. let core = &mut core; assignments_key.vrf_sign_extra_after_check( - relay_vrf_modulo_transcript(relay_vrf_story.clone(), rvm_sample), + relay_vrf_modulo_transcript_v1(relay_vrf_story.clone(), rvm_sample), |vrf_in_out| { *core = relay_vrf_modulo_core(&vrf_in_out, config.n_cores); if let Some((candidate_hash, _)) = @@ -357,15 +453,15 @@ fn compute_relay_vrf_modulo_assignments( // has been executed. let cert = AssignmentCert { kind: AssignmentCertKind::RelayVRFModulo { sample: rvm_sample }, - vrf: approval_types::VrfSignature { - output: approval_types::VrfOutput(vrf_in_out.to_output()), - proof: approval_types::VrfProof(vrf_proof), + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), }, }; // All assignments of type RelayVRFModulo have tranche 0. assignments.entry(core).or_insert(OurAssignment { - cert, + cert: cert.into(), tranche: 0, validator_index, triggered: false, @@ -374,6 +470,84 @@ fn compute_relay_vrf_modulo_assignments( } } +fn assigned_cores_transcript(core_bitfield: &CoreBitfield) -> Transcript { + let mut t = Transcript::new(approval_types::v2::ASSIGNED_CORE_CONTEXT); + core_bitfield.using_encoded(|s| t.append_message(b"cores", s)); + t +} + +fn compute_relay_vrf_modulo_assignments_v2( + assignments_key: &schnorrkel::Keypair, + validator_index: ValidatorIndex, + config: &Config, + relay_vrf_story: RelayVRFStory, + leaving_cores: Vec<(CandidateHash, CoreIndex)>, + assignments: &mut HashMap, +) { + let mut assigned_cores = Vec::new(); + let leaving_cores = leaving_cores.iter().map(|(_, core)| core).collect::>(); + + let maybe_assignment = { + let assigned_cores = &mut assigned_cores; + assignments_key.vrf_sign_extra_after_check( + relay_vrf_modulo_transcript_v2(relay_vrf_story.clone()), + |vrf_in_out| { + *assigned_cores = relay_vrf_modulo_cores( + &vrf_in_out, + config.relay_vrf_modulo_samples, + config.n_cores, + ) + .into_iter() + .filter(|core| leaving_cores.contains(&core)) + .collect::>(); + + if !assigned_cores.is_empty() { + gum::trace!( + target: LOG_TARGET, + ?assigned_cores, + ?validator_index, + tranche = 0, + "RelayVRFModuloCompact Assignment." + ); + + let assignment_bitfield: CoreBitfield = assigned_cores + .clone() + .try_into() + .expect("Just checked `!assigned_cores.is_empty()`; qed"); + + Some(assigned_cores_transcript(&assignment_bitfield)) + } else { + None + } + }, + ) + }; + + if let Some(assignment) = maybe_assignment.map(|(vrf_in_out, vrf_proof, _)| { + let assignment_bitfield: CoreBitfield = assigned_cores + .clone() + .try_into() + .expect("Just checked `!assigned_cores.is_empty()`; qed"); + + let cert = AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: assignment_bitfield.clone(), + }, + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), + }, + }; + + // All assignments of type RelayVRFModulo have tranche 0. + OurAssignment { cert, tranche: 0, validator_index, triggered: false } + }) { + for core_index in assigned_cores { + assignments.insert(core_index, assignment.clone()); + } + } +} + fn compute_relay_vrf_delay_assignments( assignments_key: &schnorrkel::Keypair, validator_index: ValidatorIndex, @@ -392,11 +566,11 @@ fn compute_relay_vrf_delay_assignments( config.zeroth_delay_tranche_width, ); - let cert = AssignmentCert { - kind: AssignmentCertKind::RelayVRFDelay { core_index: core }, - vrf: approval_types::VrfSignature { - output: approval_types::VrfOutput(vrf_in_out.to_output()), - proof: approval_types::VrfProof(vrf_proof), + let cert = AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFDelay { core_index: core }, + vrf: VrfSignature { + output: VrfOutput(vrf_in_out.to_output()), + proof: VrfProof(vrf_proof), }, }; @@ -453,12 +627,15 @@ pub(crate) enum InvalidAssignmentReason { VRFModuloOutputMismatch, VRFDelayCoreIndexMismatch, VRFDelayOutputMismatch, + InvalidArguments, + /// Assignment vrf check resulted in 0 assigned cores. + NullAssignment, } /// Checks the crypto of an assignment cert. Failure conditions: /// * Validator index out of bounds /// * VRF signature check fails -/// * VRF output doesn't match assigned core +/// * VRF output doesn't match assigned cores /// * Core is not covered by extra data in signature /// * Core index out of bounds /// * Sample is out of bounds @@ -467,12 +644,12 @@ pub(crate) enum InvalidAssignmentReason { /// This function does not check whether the core is actually a valid assignment or not. That should /// be done outside the scope of this function. pub(crate) fn check_assignment_cert( - claimed_core_index: CoreIndex, + claimed_core_indices: CoreBitfield, validator_index: ValidatorIndex, config: &Config, relay_vrf_story: RelayVRFStory, - assignment: &AssignmentCert, - backing_group: GroupIndex, + assignment: &AssignmentCertV2, + backing_groups: Vec, ) -> Result { use InvalidAssignmentReason as Reason; @@ -484,52 +661,133 @@ pub(crate) fn check_assignment_cert( let public = schnorrkel::PublicKey::from_bytes(validator_public.as_slice()) .map_err(|_| InvalidAssignment(Reason::InvalidAssignmentKey))?; - if claimed_core_index.0 >= config.n_cores { - return Err(InvalidAssignment(Reason::CoreIndexOutOfBounds)) + // Check that we have all backing groups for claimed cores. + if claimed_core_indices.count_ones() == 0 || + claimed_core_indices.count_ones() != backing_groups.len() + { + return Err(InvalidAssignment(Reason::InvalidArguments)) } // Check that the validator was not part of the backing group // and not already assigned. - let is_in_backing = - is_in_backing_group(&config.validator_groups, validator_index, backing_group); + for (claimed_core, backing_group) in claimed_core_indices.iter_ones().zip(backing_groups.iter()) + { + if claimed_core >= config.n_cores as usize { + return Err(InvalidAssignment(Reason::CoreIndexOutOfBounds)) + } - if is_in_backing { - return Err(InvalidAssignment(Reason::IsInBackingGroup)) + let is_in_backing = + is_in_backing_group(&config.validator_groups, validator_index, *backing_group); + + if is_in_backing { + return Err(InvalidAssignment(Reason::IsInBackingGroup)) + } } - let vrf_signature = &assignment.vrf; - match assignment.kind { - AssignmentCertKind::RelayVRFModulo { sample } => { - if sample >= config.relay_vrf_modulo_samples { + let vrf_output = &assignment.vrf.output; + let vrf_proof = &assignment.vrf.proof; + let first_claimed_core_index = + claimed_core_indices.first_one().expect("Checked above; qed") as u32; + + match &assignment.kind { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => { + // Check that claimed core bitfield match the one from certificate. + if &claimed_core_indices != core_bitfield { + return Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) + } + + let (vrf_in_out, _) = public + .vrf_verify_extra( + relay_vrf_modulo_transcript_v2(relay_vrf_story), + &vrf_output.0, + &vrf_proof.0, + assigned_cores_transcript(core_bitfield), + ) + .map_err(|_| InvalidAssignment(Reason::VRFModuloOutputMismatch))?; + + let resulting_cores = relay_vrf_modulo_cores( + &vrf_in_out, + config.relay_vrf_modulo_samples, + config.n_cores, + ); + + // Currently validators can opt out of checking specific cores. + // This is the same issue to how validator can opt out and not send their assignments in + // the first place. Ensure that the `vrf_in_out` actually includes all of the claimed + // cores. + for claimed_core_index in claimed_core_indices.iter_ones() { + if !resulting_cores.contains(&CoreIndex(claimed_core_index as u32)) { + gum::debug!( + target: LOG_TARGET, + ?resulting_cores, + ?claimed_core_indices, + vrf_modulo_cores = ?resulting_cores, + "Assignment claimed cores mismatch", + ); + return Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) + } + } + + Ok(0) + }, + AssignmentCertKindV2::RelayVRFModulo { sample } => { + if *sample >= config.relay_vrf_modulo_samples { return Err(InvalidAssignment(Reason::SampleOutOfBounds)) } + // Enforce claimed candidates is 1. + if claimed_core_indices.count_ones() != 1 { + gum::warn!( + target: LOG_TARGET, + ?claimed_core_indices, + "`RelayVRFModulo` assignment must always claim 1 core", + ); + return Err(InvalidAssignment(Reason::InvalidArguments)) + } + let (vrf_in_out, _) = public .vrf_verify_extra( - relay_vrf_modulo_transcript(relay_vrf_story, sample), - &vrf_signature.output.0, - &vrf_signature.proof.0, - assigned_core_transcript(claimed_core_index), + relay_vrf_modulo_transcript_v1(relay_vrf_story, *sample), + &vrf_output.0, + &vrf_proof.0, + assigned_core_transcript(CoreIndex(first_claimed_core_index)), ) .map_err(|_| InvalidAssignment(Reason::VRFModuloOutputMismatch))?; + let core = relay_vrf_modulo_core(&vrf_in_out, config.n_cores); // ensure that the `vrf_in_out` actually gives us the claimed core. - if relay_vrf_modulo_core(&vrf_in_out, config.n_cores) == claimed_core_index { + if core.0 == first_claimed_core_index { Ok(0) } else { + gum::debug!( + target: LOG_TARGET, + ?core, + ?claimed_core_indices, + "Assignment claimed cores mismatch", + ); Err(InvalidAssignment(Reason::VRFModuloCoreIndexMismatch)) } }, - AssignmentCertKind::RelayVRFDelay { core_index } => { - if core_index != claimed_core_index { + AssignmentCertKindV2::RelayVRFDelay { core_index } => { + // Enforce claimed candidates is 1. + if claimed_core_indices.count_ones() != 1 { + gum::debug!( + target: LOG_TARGET, + ?claimed_core_indices, + "`RelayVRFDelay` assignment must always claim 1 core", + ); + return Err(InvalidAssignment(Reason::InvalidArguments)) + } + + if core_index.0 != first_claimed_core_index { return Err(InvalidAssignment(Reason::VRFDelayCoreIndexMismatch)) } let (vrf_in_out, _) = public .vrf_verify( - relay_vrf_delay_transcript(relay_vrf_story, core_index), - &vrf_signature.output.0, - &vrf_signature.proof.0, + relay_vrf_delay_transcript(relay_vrf_story, *core_index), + &vrf_output.0, + &vrf_proof.0, ) .map_err(|_| InvalidAssignment(Reason::VRFDelayOutputMismatch))?; @@ -550,6 +808,19 @@ fn is_in_backing_group( validator_groups.get(group).map_or(false, |g| g.contains(&validator)) } +/// Migration helpers. +impl From for OurAssignment { + fn from(value: crate::approval_db::v1::OurAssignment) -> Self { + Self { + cert: value.cert.into(), + tranche: value.tranche, + validator_index: value.validator_index, + // Whether the assignment has been triggered already. + triggered: value.triggered, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -630,10 +901,11 @@ mod tests { ]), n_cores: 2, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![(c_a, CoreIndex(0), GroupIndex(1)), (c_b, CoreIndex(1), GroupIndex(0))], + false, ); // Note that alice is in group 0, which was the backing group for core 1. @@ -665,10 +937,11 @@ mod tests { ]), n_cores: 2, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![(c_a, CoreIndex(0), GroupIndex(0)), (c_b, CoreIndex(1), GroupIndex(1))], + false, ); assert_eq!(assignments.len(), 1); @@ -692,19 +965,21 @@ mod tests { validator_groups: Default::default(), n_cores: 0, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 10, n_delay_tranches: 40, }, vec![], + false, ); assert!(assignments.is_empty()); } + #[derive(Debug)] struct MutatedAssignment { - core: CoreIndex, - cert: AssignmentCert, - group: GroupIndex, + cores: CoreBitfield, + cert: AssignmentCertV2, + groups: Vec, own_group: GroupIndex, val_index: ValidatorIndex, config: Config, @@ -729,12 +1004,12 @@ mod tests { validator_groups: basic_groups(n_validators, n_cores), n_cores: n_cores as u32, zeroth_delay_tranche_width: 10, - relay_vrf_modulo_samples: 3, + relay_vrf_modulo_samples: 15, n_delay_tranches: 40, }; let relay_vrf_story = RelayVRFStory([42u8; 32]); - let assignments = compute_assignments( + let mut assignments = compute_assignments( &keystore, relay_vrf_story.clone(), &config, @@ -747,19 +1022,42 @@ mod tests { ) }) .collect::>(), + false, ); + // Extend with v2 assignments as well + assignments.extend(compute_assignments( + &keystore, + relay_vrf_story.clone(), + &config, + (0..n_cores) + .map(|i| { + ( + CandidateHash(Hash::repeat_byte(i as u8)), + CoreIndex(i as u32), + group_for_core(i), + ) + }) + .collect::>(), + true, + )); + let mut counted = 0; for (core, assignment) in assignments { + let cores = match assignment.cert.kind.clone() { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => core_bitfield, + AssignmentCertKindV2::RelayVRFModulo { sample: _ } => core.into(), + AssignmentCertKindV2::RelayVRFDelay { core_index } => core_index.into(), + }; + let mut mutated = MutatedAssignment { - core, - group: group_for_core(core.0 as _), + cores: cores.clone(), + groups: cores.iter_ones().map(|core| group_for_core(core)).collect(), cert: assignment.cert, own_group: GroupIndex(0), val_index: ValidatorIndex(0), config: config.clone(), }; - let expected = match f(&mut mutated) { None => continue, Some(e) => e, @@ -768,16 +1066,16 @@ mod tests { counted += 1; let is_good = check_assignment_cert( - mutated.core, + mutated.cores, mutated.val_index, &mutated.config, relay_vrf_story.clone(), &mutated.cert, - mutated.group, + mutated.groups, ) .is_ok(); - assert_eq!(expected, is_good) + assert_eq!(expected, is_good); } assert!(counted > 0); @@ -791,7 +1089,7 @@ mod tests { #[test] fn check_rejects_claimed_core_out_of_bounds() { check_mutated_assignments(200, 100, 25, |m| { - m.core.0 += 100; + m.cores = CoreIndex(100).into(); Some(false) }); } @@ -799,7 +1097,7 @@ mod tests { #[test] fn check_rejects_in_backing_group() { check_mutated_assignments(200, 100, 25, |m| { - m.group = m.own_group; + m.groups[0] = m.own_group; Some(false) }); } @@ -815,9 +1113,10 @@ mod tests { #[test] fn check_rejects_delay_bad_vrf() { check_mutated_assignments(40, 10, 8, |m| { + let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFDelay { .. } => { - m.cert.vrf = garbage_vrf_signature(); + AssignmentCertKindV2::RelayVRFDelay { .. } => { + m.cert.vrf = vrf_signature; Some(false) }, _ => None, // skip everything else. @@ -828,9 +1127,14 @@ mod tests { #[test] fn check_rejects_modulo_bad_vrf() { check_mutated_assignments(200, 100, 25, |m| { + let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { .. } => { - m.cert.vrf = garbage_vrf_signature(); + AssignmentCertKindV2::RelayVRFModulo { .. } => { + m.cert.vrf = vrf_signature; + Some(false) + }, + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => { + m.cert.vrf = vrf_signature; Some(false) }, _ => None, // skip everything else. @@ -842,10 +1146,11 @@ mod tests { fn check_rejects_modulo_sample_out_of_bounds() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { sample } => { + AssignmentCertKindV2::RelayVRFModulo { sample } => { m.config.relay_vrf_modulo_samples = sample; Some(false) }, + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield: _ } => Some(true), _ => None, // skip everything else. } }); @@ -855,8 +1160,11 @@ mod tests { fn check_rejects_delay_claimed_core_wrong() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFDelay { .. } => { - m.core = CoreIndex((m.core.0 + 1) % 100); + AssignmentCertKindV2::RelayVRFDelay { .. } => { + // for core in &mut m.cores { + // core.0 = (core.0 + 1) % 100; + // } + m.cores = CoreIndex((m.cores.first_one().unwrap() + 1) as u32 % 100).into(); Some(false) }, _ => None, // skip everything else. @@ -868,8 +1176,10 @@ mod tests { fn check_rejects_modulo_core_wrong() { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { - AssignmentCertKind::RelayVRFModulo { .. } => { - m.core = CoreIndex((m.core.0 + 1) % 100); + AssignmentCertKindV2::RelayVRFModulo { .. } | + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => { + m.cores = CoreIndex((m.cores.first_one().unwrap() + 1) as u32 % 100).into(); + Some(false) }, _ => None, // skip everything else. diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index c504ba71b3c2..4345380ed2f9 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -30,7 +30,10 @@ use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ - approval::{self as approval_types, BlockApprovalMeta, RelayVRFStory}, + approval::{ + self as approval_types, + v1::{BlockApprovalMeta, RelayVRFStory}, + }, MAX_FINALITY_LAG, }; use polkadot_node_subsystem::{ @@ -53,7 +56,7 @@ use futures::{channel::oneshot, prelude::*}; use std::collections::HashMap; -use super::approval_db::v1; +use super::approval_db::v2; use crate::{ backend::{Backend, OverlayedBackend}, criteria::{AssignmentCriteria, OurAssignment}, @@ -92,7 +95,7 @@ enum ImportedBlockInfoError { FutureCancelled(&'static str, futures::channel::oneshot::Canceled), #[error(transparent)] - ApprovalError(approval_types::ApprovalError), + ApprovalError(approval_types::v1::ApprovalError), #[error("block is already finalized")] BlockAlreadyFinalized, @@ -216,7 +219,7 @@ async fn imported_block_info( .ok_or(ImportedBlockInfoError::SessionInfoUnavailable)?; let (assignments, slot, relay_vrf_story) = { - let unsafe_vrf = approval_types::babe_unsafe_vrf_info(&block_header); + let unsafe_vrf = approval_types::v1::babe_unsafe_vrf_info(&block_header); match unsafe_vrf { Some(unsafe_vrf) => { @@ -497,7 +500,7 @@ pub(crate) async fn handle_new_head( ctx.send_message(ChainSelectionMessage::Approved(block_hash)).await; } - let block_entry = v1::BlockEntry { + let block_entry = v2::BlockEntry { block_hash, parent_hash: block_header.parent_hash, block_number: block_header.number, @@ -510,6 +513,7 @@ pub(crate) async fn handle_new_head( .collect(), approved_bitfield, children: Vec::new(), + distributed_assignments: Default::default(), }; gum::trace!( @@ -588,11 +592,11 @@ pub(crate) async fn handle_new_head( #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::{approval_db::v1::DbBackend, RuntimeInfo, RuntimeInfoConfig}; + use crate::{approval_db::v2::DbBackend, RuntimeInfo, RuntimeInfoConfig}; use ::test_helpers::{dummy_candidate_receipt, dummy_hash}; use assert_matches::assert_matches; use polkadot_node_primitives::{ - approval::{VrfSignature, VrfTranscript}, + approval::v1::{VrfSignature, VrfTranscript}, DISPUTE_WINDOW, }; use polkadot_node_subsystem::messages::{AllMessages, ApprovalVotingMessage}; @@ -608,7 +612,7 @@ pub(crate) mod tests { pub(crate) use sp_runtime::{Digest, DigestItem}; use std::{pin::Pin, sync::Arc}; - use crate::{approval_db::v1::Config as DatabaseConfig, criteria, BlockEntry}; + use crate::{approval_db::v2::Config as DatabaseConfig, criteria, BlockEntry}; const DATA_COL: u32 = 0; @@ -654,7 +658,7 @@ pub(crate) mod tests { fn compute_assignments( &self, _keystore: &LocalKeystore, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _config: &criteria::Config, _leaving_cores: Vec<( CandidateHash, @@ -667,13 +671,14 @@ pub(crate) mod tests { fn check_assignment_cert( &self, - _claimed_core_index: polkadot_primitives::CoreIndex, + _claimed_core_bitfield: polkadot_node_primitives::approval::v2::CoreBitfield, _validator_index: polkadot_primitives::ValidatorIndex, _config: &criteria::Config, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, - _assignment: &polkadot_node_primitives::approval::AssignmentCert, - _backing_group: polkadot_primitives::GroupIndex, - ) -> Result { + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, + _backing_groups: Vec, + ) -> Result + { Ok(0) } } @@ -1252,7 +1257,7 @@ pub(crate) mod tests { let (state, mut session_info_provider) = single_session_state(); overlay_db.write_block_entry( - v1::BlockEntry { + v2::BlockEntry { block_hash: parent_hash, parent_hash: Default::default(), block_number: 4, @@ -1262,6 +1267,7 @@ pub(crate) mod tests { candidates: Vec::new(), approved_bitfield: Default::default(), children: Vec::new(), + distributed_assignments: Default::default(), } .into(), ); @@ -1294,7 +1300,7 @@ pub(crate) mod tests { // the first candidate should be insta-approved // the second should not let entry: BlockEntry = - v1::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) + v2::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) .unwrap() .unwrap() .into(); diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index b29e47b4c435..beaca43ee528 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -25,7 +25,11 @@ use jaeger::{hash_to_trace_identifier, PerLeafSpan}; use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ approval::{ - BlockApprovalMeta, DelayTranche, IndirectAssignmentCert, IndirectSignedApprovalVote, + v1::{BlockApprovalMeta, DelayTranche, IndirectSignedApprovalVote}, + v2::{ + AssignmentCertKindV2, BitfieldError, CandidateBitfield, CoreBitfield, + IndirectAssignmentCertV2, + }, }, ValidationResult, DISPUTE_WINDOW, }; @@ -75,12 +79,13 @@ use std::{ }; use approval_checking::RequiredTranches; +use bitvec::{order::Lsb0, vec::BitVec}; use criteria::{AssignmentCriteria, RealAssignmentCriteria}; use persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}; use time::{slot_number_to_tick, Clock, ClockExt, SystemClock, Tick}; mod approval_checking; -mod approval_db; +pub mod approval_db; mod backend; mod criteria; mod import; @@ -89,8 +94,9 @@ mod persisted_entries; mod time; use crate::{ - approval_db::v1::{Config as DatabaseConfig, DbBackend}, + approval_db::v2::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, + criteria::InvalidAssignmentReason, }; #[cfg(test)] @@ -109,7 +115,7 @@ const APPROVAL_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(1024) { const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. const APPROVAL_DELAY: Tick = 2; -const LOG_TARGET: &str = "parachain::approval-voting"; +pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; /// Configuration for the approval voting subsystem #[derive(Debug, Clone)] @@ -379,8 +385,8 @@ impl ApprovalVotingSubsystem { /// The operation is not allowed for blocks older than the last finalized one. pub fn revert_to(&self, hash: Hash) -> Result<(), SubsystemError> { let config = - approval_db::v1::Config { col_approval_data: self.db_config.col_approval_data }; - let mut backend = approval_db::v1::DbBackend::new(self.db.clone(), config); + approval_db::v2::Config { col_approval_data: self.db_config.col_approval_data }; + let mut backend = approval_db::v2::DbBackend::new(self.db.clone(), config); let mut overlay = OverlayedBackend::new(&backend); ops::revert_to(&mut overlay, hash)?; @@ -741,14 +747,15 @@ enum Action { tick: Tick, }, LaunchApproval { + claimed_candidate_indices: CandidateBitfield, candidate_hash: CandidateHash, - indirect_cert: IndirectAssignmentCert, + indirect_cert: IndirectAssignmentCertV2, assignment_tranche: DelayTranche, relay_block_hash: Hash, - candidate_index: CandidateIndex, session: SessionIndex, candidate: CandidateReceipt, backing_group: GroupIndex, + distribute_assignment: bool, }, NoteApprovedInChainSelection(Hash), IssueApproval(CandidateHash, ApprovalVoteRequest), @@ -963,14 +970,15 @@ async fn handle_actions( actions_iter = next_actions.into_iter(); }, Action::LaunchApproval { + claimed_candidate_indices, candidate_hash, indirect_cert, assignment_tranche, relay_block_hash, - candidate_index, session, candidate, backing_group, + distribute_assignment, } => { // Don't launch approval work if the node is syncing. if let Mode::Syncing(_) = *mode { @@ -991,10 +999,12 @@ async fn handle_actions( launch_approval_span.add_string_tag("block-hash", format!("{:?}", block_hash)); let validator_index = indirect_cert.validator; - ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( - indirect_cert, - candidate_index, - )); + if distribute_assignment { + ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( + indirect_cert, + claimed_candidate_indices, + )); + } match approvals_cache.get(&candidate_hash) { Some(ApprovalOutcome::Approved) => { @@ -1061,6 +1071,49 @@ async fn handle_actions( Ok(conclude) } +fn cores_to_candidate_indices( + core_indices: &CoreBitfield, + block_entry: &BlockEntry, +) -> Result { + let mut candidate_indices = Vec::new(); + + // Map from core index to candidate index. + for claimed_core_index in core_indices.iter_ones() { + if let Some(candidate_index) = block_entry + .candidates() + .iter() + .position(|(core_index, _)| core_index.0 == claimed_core_index as u32) + { + candidate_indices.push(candidate_index as CandidateIndex); + } + } + + CandidateBitfield::try_from(candidate_indices) +} + +// Returns the claimed core bitfield from the assignment cert, the candidate hash and a +// `BlockEntry`. Can fail only for VRF Delay assignments for which we cannot find the candidate hash +// in the block entry which indicates a bug or corrupted storage. +fn get_assignment_core_indices( + assignment: &AssignmentCertKindV2, + candidate_hash: &CandidateHash, + block_entry: &BlockEntry, +) -> Option { + match &assignment { + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + Some(core_bitfield.clone()), + AssignmentCertKindV2::RelayVRFModulo { sample: _ } => block_entry + .candidates() + .iter() + .find(|(_core_index, h)| candidate_hash == h) + .map(|(core_index, _candidate_hash)| { + CoreBitfield::try_from(vec![*core_index]).expect("Not an empty vec; qed") + }), + AssignmentCertKindV2::RelayVRFDelay { core_index } => + Some(CoreBitfield::try_from(vec![*core_index]).expect("Not an empty vec; qed")), + } +} + fn distribution_messages_for_activation( db: &OverlayedBackend<'_, impl Backend>, state: &State, @@ -1125,33 +1178,95 @@ fn distribution_messages_for_activation( match approval_entry.local_statements() { (None, None) | (None, Some(_)) => {}, // second is impossible case. (Some(assignment), None) => { - messages.push(ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCert { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - i as _, - )); + if let Some(claimed_core_indices) = get_assignment_core_indices( + &assignment.cert().kind, + &candidate_hash, + &block_entry, + ) { + match cores_to_candidate_indices( + &claimed_core_indices, + &block_entry, + ) { + Ok(bitfield) => messages.push( + ApprovalDistributionMessage::DistributeAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }, + bitfield, + ), + ), + Err(err) => { + // Should never happen. If we fail here it means the + // assignment is null (no cores claimed). + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + ?err, + "Failed to create assignment bitfield", + ); + }, + } + } else { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + "Cannot get assignment claimed core indices", + ); + } }, (Some(assignment), Some(approval_sig)) => { - messages.push(ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCert { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - i as _, - )); + if let Some(claimed_core_indices) = get_assignment_core_indices( + &assignment.cert().kind, + &candidate_hash, + &block_entry, + ) { + match cores_to_candidate_indices( + &claimed_core_indices, + &block_entry, + ) { + Ok(bitfield) => messages.push( + ApprovalDistributionMessage::DistributeAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }, + bitfield, + ), + ), + Err(err) => { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + ?err, + "Failed to create assignment bitfield", + ); + // If we didn't send assignment, we don't send approval. + continue + }, + } - messages.push(ApprovalDistributionMessage::DistributeApproval( - IndirectSignedApprovalVote { - block_hash, - candidate_index: i as _, - validator: assignment.validator_index(), - signature: approval_sig, - }, - )) + messages.push(ApprovalDistributionMessage::DistributeApproval( + IndirectSignedApprovalVote { + block_hash, + candidate_index: i as _, + validator: assignment.validator_index(), + signature: approval_sig, + }, + )); + } else { + gum::warn!( + target: LOG_TARGET, + ?block_hash, + ?candidate_hash, + "Cannot get assignment claimed core indices", + ); + } }, } }, @@ -1271,14 +1386,14 @@ async fn handle_from_overseer( vec![Action::Conclude] }, FromOrchestra::Communication { msg } => match msg { - ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_core, res) => { + ApprovalVotingMessage::CheckAndImportAssignment(a, claimed_cores, res) => { let (check_outcome, actions) = check_and_import_assignment( ctx.sender(), state, db, session_info_provider, a, - claimed_core, + claimed_cores, ) .await?; let _ = res.send(check_outcome); @@ -1448,7 +1563,6 @@ async fn handle_approved_ancestor( let mut span = span .child("handle-approved-ancestor") .with_stage(jaeger::Stage::ApprovalChecking); - use bitvec::{order::Lsb0, vec::BitVec}; let mut all_approved_max = None; @@ -1787,8 +1901,8 @@ async fn check_and_import_assignment( state: &State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, - assignment: IndirectAssignmentCert, - candidate_index: CandidateIndex, + assignment: IndirectAssignmentCertV2, + candidate_indices: CandidateBitfield, ) -> SubsystemResult<(AssignmentCheckResult, Vec)> where Sender: SubsystemSender, @@ -1801,9 +1915,12 @@ where .map(|span| span.child("check-and-import-assignment")) .unwrap_or_else(|| jaeger::Span::new(assignment.block_hash, "check-and-import-assignment")) .with_relay_parent(assignment.block_hash) - .with_uint_tag("candidate-index", candidate_index as u64) .with_stage(jaeger::Stage::ApprovalChecking); + for candidate_index in candidate_indices.iter_ones() { + check_and_import_assignment_span.add_uint_tag("candidate-index", candidate_index as u64); + } + let block_entry = match db.load_block_entry(&assignment.block_hash)? { Some(b) => b, None => @@ -1833,39 +1950,64 @@ where )), }; - let (claimed_core_index, assigned_candidate_hash) = - match block_entry.candidate(candidate_index as usize) { - Some((c, h)) => (*c, *h), + let n_cores = session_info.n_cores as usize; + + // Early check the candidate bitfield and core bitfields lengths < `n_cores`. + // Core bitfield length is checked later in `check_assignment_cert`. + if candidate_indices.len() > n_cores { + gum::debug!( + target: LOG_TARGET, + validator = assignment.validator.0, + n_cores, + candidate_bitfield_len = ?candidate_indices.len(), + "Oversized bitfield", + ); + + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield( + candidate_indices.len(), + )), + Vec::new(), + )) + } + + // The Compact VRF modulo assignment cert has multiple core assignments. + let mut backing_groups = Vec::new(); + let mut claimed_core_indices = Vec::new(); + let mut assigned_candidate_hashes = Vec::new(); + + for candidate_index in candidate_indices.iter_ones() { + let (claimed_core_index, assigned_candidate_hash) = + match block_entry.candidate(candidate_index) { + Some((c, h)) => (*c, *h), + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex( + candidate_index as _, + )), + Vec::new(), + )), // no candidate at core. + }; + + let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { + Some(c) => c, None => return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex( - candidate_index, + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( + candidate_index as _, + assigned_candidate_hash, )), Vec::new(), )), // no candidate at core. }; - check_and_import_assignment_span - .add_string_tag("candidate-hash", format!("{:?}", assigned_candidate_hash)); - check_and_import_assignment_span.add_string_tag( - "traceID", - format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), - ); - - let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { - Some(c) => c, - None => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( - candidate_index, - assigned_candidate_hash, - )), - Vec::new(), - )), - }; + check_and_import_assignment_span + .add_string_tag("candidate-hash", format!("{:?}", assigned_candidate_hash)); + check_and_import_assignment_span.add_string_tag( + "traceID", + format!("{:?}", jaeger::hash_to_trace_identifier(assigned_candidate_hash.0)), + ); - let res = { - // import the assignment. let approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { Some(a) => a, None => @@ -1878,79 +2020,144 @@ where )), }; - let res = state.assignment_criteria.check_assignment_cert( - claimed_core_index, - assignment.validator, - &criteria::Config::from(session_info), - block_entry.relay_vrf_story(), - &assignment.cert, - approval_entry.backing_group(), - ); + backing_groups.push(approval_entry.backing_group()); + claimed_core_indices.push(claimed_core_index); + assigned_candidate_hashes.push(assigned_candidate_hash); + } - let tranche = match res { - Err(crate::criteria::InvalidAssignment(reason)) => - return Ok(( - AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( - assignment.validator, - format!("{:?}", reason), - )), - Vec::new(), + // Error on null assignments. + if claimed_core_indices.is_empty() { + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( + assignment.validator, + format!("{:?}", InvalidAssignmentReason::NullAssignment), + )), + Vec::new(), + )) + } + + // Check the assignment certificate. + let res = state.assignment_criteria.check_assignment_cert( + claimed_core_indices + .clone() + .try_into() + .expect("Checked for null assignment above; qed"), + assignment.validator, + &criteria::Config::from(session_info), + block_entry.relay_vrf_story(), + &assignment.cert, + backing_groups, + ); + + let tranche = match res { + Err(crate::criteria::InvalidAssignment(reason)) => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert( + assignment.validator, + format!("{:?}", reason), )), - Ok(tranche) => { - let current_tranche = - state.clock.tranche_now(state.slot_duration_millis, block_entry.slot()); + Vec::new(), + )), + Ok(tranche) => { + let current_tranche = + state.clock.tranche_now(state.slot_duration_millis, block_entry.slot()); - let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; + let too_far_in_future = current_tranche + TICK_TOO_FAR_IN_FUTURE as DelayTranche; - if tranche >= too_far_in_future { - return Ok((AssignmentCheckResult::TooFarInFuture, Vec::new())) - } + if tranche >= too_far_in_future { + return Ok((AssignmentCheckResult::TooFarInFuture, Vec::new())) + } - tranche - }, - }; + tranche + }, + }; - check_and_import_assignment_span.add_uint_tag("tranche", tranche as u64); + let mut actions = Vec::new(); + let res = { + let mut is_duplicate = true; + // Import the assignments for all cores in the cert. + for (assigned_candidate_hash, candidate_index) in + assigned_candidate_hashes.iter().zip(candidate_indices.iter_ones()) + { + let mut candidate_entry = match db.load_candidate_entry(&assigned_candidate_hash)? { + Some(c) => c, + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidate( + candidate_index as _, + *assigned_candidate_hash, + )), + Vec::new(), + )), + }; - let is_duplicate = approval_entry.is_assigned(assignment.validator); - approval_entry.import_assignment(tranche, assignment.validator, tick_now); + let approval_entry = match candidate_entry.approval_entry_mut(&assignment.block_hash) { + Some(a) => a, + None => + return Ok(( + AssignmentCheckResult::Bad(AssignmentCheckError::Internal( + assignment.block_hash, + *assigned_candidate_hash, + )), + Vec::new(), + )), + }; + is_duplicate &= approval_entry.is_assigned(assignment.validator); + approval_entry.import_assignment(tranche, assignment.validator, tick_now); + check_and_import_assignment_span.add_uint_tag("tranche", tranche as u64); + + // We've imported a new assignment, so we need to schedule a wake-up for when that might + // no-show. + if let Some((approval_entry, status)) = state + .approval_status(sender, session_info_provider, &block_entry, &candidate_entry) + .await + { + actions.extend(schedule_wakeup_action( + approval_entry, + block_entry.block_hash(), + block_entry.block_number(), + *assigned_candidate_hash, + status.block_tick, + tick_now, + status.required_tranches, + )); + } + + // We also write the candidate entry as it now contains the new candidate. + db.write_candidate_entry(candidate_entry.into()); + } + // Since we don't account for tranche in distribution message fingerprinting, some + // validators can be assigned to the same core (VRF modulo vs VRF delay). These can be + // safely ignored ignored. However, if an assignment is for multiple cores (these are only + // tranche0), we cannot ignore it, because it would mean ignoring other non duplicate + // assignments. if is_duplicate { AssignmentCheckResult::AcceptedDuplicate + } else if candidate_indices.count_ones() > 1 { + gum::trace!( + target: LOG_TARGET, + validator = assignment.validator.0, + candidate_hashes = ?assigned_candidate_hashes, + assigned_cores = ?claimed_core_indices, + ?tranche, + "Imported assignments for multiple cores.", + ); + + AssignmentCheckResult::Accepted } else { gum::trace!( target: LOG_TARGET, validator = assignment.validator.0, - candidate_hash = ?assigned_candidate_hash, - para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, - "Imported assignment.", + candidate_hashes = ?assigned_candidate_hashes, + assigned_cores = ?claimed_core_indices, + "Imported assignment for a single core.", ); AssignmentCheckResult::Accepted } }; - let mut actions = Vec::new(); - - // We've imported a new approval, so we need to schedule a wake-up for when that might no-show. - if let Some((approval_entry, status)) = state - .approval_status(sender, session_info_provider, &block_entry, &candidate_entry) - .await - { - actions.extend(schedule_wakeup_action( - approval_entry, - block_entry.block_hash(), - block_entry.block_number(), - assigned_candidate_hash, - status.block_tick, - tick_now, - status.required_tranches, - )); - } - - // We also write the candidate entry as it now contains the new candidate. - db.write_candidate_entry(candidate_entry.into()); - Ok((res, actions)) } @@ -2324,7 +2531,7 @@ async fn process_wakeup( let candidate_entry = db.load_candidate_entry(&candidate_hash)?; // If either is not present, we have nothing to wakeup. Might have lost a race with finality - let (block_entry, mut candidate_entry) = match (block_entry, candidate_entry) { + let (mut block_entry, mut candidate_entry) = match (block_entry, candidate_entry) { (Some(b), Some(c)) => (b, c), _ => return Ok(Vec::new()), }; @@ -2404,31 +2611,58 @@ async fn process_wakeup( if let Some((cert, val_index, tranche)) = maybe_cert { let indirect_cert = - IndirectAssignmentCert { block_hash: relay_block, validator: val_index, cert }; + IndirectAssignmentCertV2 { block_hash: relay_block, validator: val_index, cert }; + + gum::trace!( + target: LOG_TARGET, + ?candidate_hash, + para_id = ?candidate_receipt.descriptor.para_id, + block_hash = ?relay_block, + "Launching approval work.", + ); - let index_in_candidate = - block_entry.candidates().iter().position(|(_, h)| &candidate_hash == h); + if let Some(claimed_core_indices) = + get_assignment_core_indices(&indirect_cert.cert.kind, &candidate_hash, &block_entry) + { + match cores_to_candidate_indices(&claimed_core_indices, &block_entry) { + Ok(claimed_candidate_indices) => { + // Ensure we distribute multiple core assignments just once. + let distribute_assignment = if claimed_candidate_indices.count_ones() > 1 { + !block_entry.mark_assignment_distributed(claimed_candidate_indices.clone()) + } else { + true + }; + db.write_block_entry(block_entry.clone()); - if let Some(i) = index_in_candidate { - gum::trace!( + actions.push(Action::LaunchApproval { + claimed_candidate_indices, + candidate_hash, + indirect_cert, + assignment_tranche: tranche, + relay_block_hash: relay_block, + session: block_entry.session(), + candidate: candidate_receipt, + backing_group, + distribute_assignment, + }); + }, + Err(err) => { + // Never happens, it should only happen if no cores are claimed, which is a bug. + gum::warn!( + target: LOG_TARGET, + block_hash = ?relay_block, + ?err, + "Failed to create assignment bitfield" + ); + }, + }; + } else { + gum::warn!( target: LOG_TARGET, - ?candidate_hash, - para_id = ?candidate_receipt.descriptor.para_id, block_hash = ?relay_block, - "Launching approval work.", + ?candidate_hash, + "Cannot get assignment claimed core indices", ); - - // sanity: should always be present. - actions.push(Action::LaunchApproval { - candidate_hash, - indirect_cert, - assignment_tranche: tranche, - relay_block_hash: relay_block, - candidate_index: i as _, - session: block_entry.session(), - candidate: candidate_receipt, - backing_group, - }); } } // Although we checked approval earlier in this function, diff --git a/polkadot/node/core/approval-voting/src/ops.rs b/polkadot/node/core/approval-voting/src/ops.rs index 6f57b2f80e8a..a6f0ecf9d1f0 100644 --- a/polkadot/node/core/approval-voting/src/ops.rs +++ b/polkadot/node/core/approval-voting/src/ops.rs @@ -25,7 +25,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, CandidateReceipt, GroupInd use std::collections::{hash_map::Entry, BTreeMap, HashMap}; use super::{ - approval_db::v1::{OurAssignment, StoredBlockRange}, + approval_db::v2::{OurAssignment, StoredBlockRange}, backend::{Backend, OverlayedBackend}, persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}, LOG_TARGET, diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 9b6592220275..155b2f9c4e02 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -20,16 +20,21 @@ //! Within that context, things are plain-old-data. Within this module, //! data and logic are intertwined. -use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche, RelayVRFStory}; +use polkadot_node_primitives::approval::{ + v1::{DelayTranche, RelayVRFStory}, + v2::{AssignmentCertV2, CandidateBitfield}, +}; use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; -use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice, vec::BitVec}; +use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice}; use std::collections::BTreeMap; +use crate::approval_db::v2::Bitfield; + use super::{criteria::OurAssignment, time::Tick}; /// Metadata regarding a specific tranche of assignments for a specific candidate. @@ -53,8 +58,8 @@ impl TrancheEntry { } } -impl From for TrancheEntry { - fn from(entry: crate::approval_db::v1::TrancheEntry) -> Self { +impl From for TrancheEntry { + fn from(entry: crate::approval_db::v2::TrancheEntry) -> Self { TrancheEntry { tranche: entry.tranche, assignments: entry.assignments.into_iter().map(|(v, t)| (v, t.into())).collect(), @@ -62,7 +67,7 @@ impl From for TrancheEntry { } } -impl From for crate::approval_db::v1::TrancheEntry { +impl From for crate::approval_db::v2::TrancheEntry { fn from(entry: TrancheEntry) -> Self { Self { tranche: entry.tranche, @@ -80,7 +85,7 @@ pub struct ApprovalEntry { our_assignment: Option, our_approval_sig: Option, // `n_validators` bits. - assignments: BitVec, + assigned_validators: Bitfield, approved: bool, } @@ -92,10 +97,17 @@ impl ApprovalEntry { our_assignment: Option, our_approval_sig: Option, // `n_validators` bits. - assignments: BitVec, + assigned_validators: Bitfield, approved: bool, ) -> Self { - Self { tranches, backing_group, our_assignment, our_approval_sig, assignments, approved } + Self { + tranches, + backing_group, + our_assignment, + our_approval_sig, + assigned_validators, + approved, + } } // Access our assignment for this approval entry. @@ -107,7 +119,7 @@ impl ApprovalEntry { pub fn trigger_our_assignment( &mut self, tick_now: Tick, - ) -> Option<(AssignmentCert, ValidatorIndex, DelayTranche)> { + ) -> Option<(AssignmentCertV2, ValidatorIndex, DelayTranche)> { let our = self.our_assignment.as_mut().and_then(|a| { if a.triggered() { return None @@ -131,7 +143,10 @@ impl ApprovalEntry { /// Whether a validator is already assigned. pub fn is_assigned(&self, validator_index: ValidatorIndex) -> bool { - self.assignments.get(validator_index.0 as usize).map(|b| *b).unwrap_or(false) + self.assigned_validators + .get(validator_index.0 as usize) + .map(|b| *b) + .unwrap_or(false) } /// Import an assignment. No-op if already assigned on the same tranche. @@ -158,14 +173,14 @@ impl ApprovalEntry { }; self.tranches[idx].assignments.push((validator_index, tick_now)); - self.assignments.set(validator_index.0 as _, true); + self.assigned_validators.set(validator_index.0 as _, true); } // Produce a bitvec indicating the assignments of all validators up to and // including `tranche`. - pub fn assignments_up_to(&self, tranche: DelayTranche) -> BitVec { + pub fn assignments_up_to(&self, tranche: DelayTranche) -> Bitfield { self.tranches.iter().take_while(|e| e.tranche <= tranche).fold( - bitvec::bitvec![u8, BitOrderLsb0; 0; self.assignments.len()], + bitvec::bitvec![u8, BitOrderLsb0; 0; self.assigned_validators.len()], |mut a, e| { for &(v, _) in &e.assignments { a.set(v.0 as _, true); @@ -193,12 +208,12 @@ impl ApprovalEntry { /// Get the number of validators in this approval entry. pub fn n_validators(&self) -> usize { - self.assignments.len() + self.assigned_validators.len() } /// Get the number of assignments by validators, including the local validator. pub fn n_assignments(&self) -> usize { - self.assignments.count_ones() + self.assigned_validators.count_ones() } /// Get the backing group index of the approval entry. @@ -219,27 +234,27 @@ impl ApprovalEntry { } } -impl From for ApprovalEntry { - fn from(entry: crate::approval_db::v1::ApprovalEntry) -> Self { +impl From for ApprovalEntry { + fn from(entry: crate::approval_db::v2::ApprovalEntry) -> Self { ApprovalEntry { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, our_assignment: entry.our_assignment.map(Into::into), our_approval_sig: entry.our_approval_sig.map(Into::into), - assignments: entry.assignments, + assigned_validators: entry.assigned_validators, approved: entry.approved, } } } -impl From for crate::approval_db::v1::ApprovalEntry { +impl From for crate::approval_db::v2::ApprovalEntry { fn from(entry: ApprovalEntry) -> Self { Self { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, our_assignment: entry.our_assignment.map(Into::into), our_approval_sig: entry.our_approval_sig.map(Into::into), - assignments: entry.assignments, + assigned_validators: entry.assigned_validators, approved: entry.approved, } } @@ -253,7 +268,7 @@ pub struct CandidateEntry { // Assignments are based on blocks, so we need to track assignments separately // based on the block we are looking at. pub block_assignments: BTreeMap, - pub approvals: BitVec, + pub approvals: Bitfield, } impl CandidateEntry { @@ -290,8 +305,8 @@ impl CandidateEntry { } } -impl From for CandidateEntry { - fn from(entry: crate::approval_db::v1::CandidateEntry) -> Self { +impl From for CandidateEntry { + fn from(entry: crate::approval_db::v2::CandidateEntry) -> Self { CandidateEntry { candidate: entry.candidate, session: entry.session, @@ -305,7 +320,7 @@ impl From for CandidateEntry { } } -impl From for crate::approval_db::v1::CandidateEntry { +impl From for crate::approval_db::v2::CandidateEntry { fn from(entry: CandidateEntry) -> Self { Self { candidate: entry.candidate, @@ -336,8 +351,12 @@ pub struct BlockEntry { // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. // The i'th bit is `true` iff the candidate has been approved in the context of this // block. The block can be considered approved if the bitfield has all bits set to `true`. - pub approved_bitfield: BitVec, + pub approved_bitfield: Bitfield, pub children: Vec, + // A list of assignments for which wea already distributed the assignment. + // We use this to ensure we don't distribute multiple core assignments twice as we track + // individual wakeups for each core. + distributed_assignments: Bitfield, } impl BlockEntry { @@ -412,6 +431,39 @@ impl BlockEntry { pub fn parent_hash(&self) -> Hash { self.parent_hash } + + /// Mark distributed assignment for many candidate indices. + /// Returns `true` if an assignment was already distributed for the `candidates`. + pub fn mark_assignment_distributed(&mut self, candidates: CandidateBitfield) -> bool { + let bitfield = candidates.into_inner(); + let total_one_bits = self.distributed_assignments.count_ones(); + + let new_len = std::cmp::max(self.distributed_assignments.len(), bitfield.len()); + self.distributed_assignments.resize(new_len, false); + self.distributed_assignments |= bitfield; + + // If the an operation did not change our current bitfied, we return true. + let distributed = total_one_bits == self.distributed_assignments.count_ones(); + + distributed + } +} + +impl From for BlockEntry { + fn from(entry: crate::approval_db::v2::BlockEntry) -> Self { + BlockEntry { + block_hash: entry.block_hash, + parent_hash: entry.parent_hash, + block_number: entry.block_number, + session: entry.session, + slot: entry.slot, + relay_vrf_story: RelayVRFStory(entry.relay_vrf_story), + candidates: entry.candidates, + approved_bitfield: entry.approved_bitfield, + children: entry.children, + distributed_assignments: entry.distributed_assignments, + } + } } impl From for BlockEntry { @@ -426,11 +478,12 @@ impl From for BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + distributed_assignments: Default::default(), } } } -impl From for crate::approval_db::v1::BlockEntry { +impl From for crate::approval_db::v2::BlockEntry { fn from(entry: BlockEntry) -> Self { Self { block_hash: entry.block_hash, @@ -442,6 +495,36 @@ impl From for crate::approval_db::v1::BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + distributed_assignments: entry.distributed_assignments, + } + } +} + +/// Migration helpers. +impl From for CandidateEntry { + fn from(value: crate::approval_db::v1::CandidateEntry) -> Self { + Self { + approvals: value.approvals, + block_assignments: value + .block_assignments + .into_iter() + .map(|(h, ae)| (h, ae.into())) + .collect(), + candidate: value.candidate, + session: value.session, + } + } +} + +impl From for ApprovalEntry { + fn from(value: crate::approval_db::v1::ApprovalEntry) -> Self { + ApprovalEntry { + tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), + backing_group: value.backing_group, + our_assignment: value.our_assignment.map(|assignment| assignment.into()), + our_approval_sig: value.our_approval_sig, + assigned_validators: value.assignments, + approved: value.approved, } } } diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index f58e60c6a487..c2ef109ad4ca 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -15,10 +15,14 @@ // along with Polkadot. If not, see . use super::*; +use crate::backend::V1ReadBackend; use polkadot_node_primitives::{ approval::{ - AssignmentCert, AssignmentCertKind, DelayTranche, VrfOutput, VrfProof, VrfSignature, - RELAY_VRF_MODULO_CONTEXT, + v1::{ + AssignmentCert, AssignmentCertKind, DelayTranche, VrfOutput, VrfProof, VrfSignature, + RELAY_VRF_MODULO_CONTEXT, + }, + v2::{AssignmentCertKindV2, AssignmentCertV2}, }, AvailableData, BlockData, PoV, }; @@ -51,7 +55,7 @@ use std::{ }; use super::{ - approval_db::v1::StoredBlockRange, + approval_db::v2::StoredBlockRange, backend::BackendWriteOp, import::tests::{ garbage_vrf_signature, AllowedSlots, BabeEpoch, BabeEpochConfiguration, @@ -111,7 +115,7 @@ fn make_sync_oracle(val: bool) -> (Box, TestSyncOracleHan #[cfg(test)] pub mod test_constants { - use crate::approval_db::v1::Config as DatabaseConfig; + use crate::approval_db::v2::Config as DatabaseConfig; const DATA_COL: u32 = 0; pub(crate) const NUM_COLUMNS: u32 = 1; @@ -161,7 +165,7 @@ impl Clock for MockClock { // This mock clock allows us to manipulate the time and // be notified when wakeups have been triggered. -#[derive(Default)] +#[derive(Default, Debug)] struct MockClockInner { tick: Tick, wakeups: Vec<(Tick, oneshot::Sender<()>)>, @@ -231,7 +235,7 @@ where fn compute_assignments( &self, _keystore: &LocalKeystore, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, _config: &criteria::Config, _leaving_cores: Vec<( CandidateHash, @@ -244,13 +248,13 @@ where fn check_assignment_cert( &self, - _claimed_core_index: polkadot_primitives::CoreIndex, + _claimed_core_bitfield: polkadot_node_primitives::approval::v2::CoreBitfield, validator_index: ValidatorIndex, _config: &criteria::Config, - _relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory, - _assignment: &polkadot_node_primitives::approval::AssignmentCert, - _backing_group: polkadot_primitives::GroupIndex, - ) -> Result { + _relay_vrf_story: polkadot_node_primitives::approval::v1::RelayVRFStory, + _assignment: &polkadot_node_primitives::approval::v2::AssignmentCertV2, + _backing_groups: Vec, + ) -> Result { self.1(validator_index) } } @@ -271,6 +275,18 @@ struct TestStoreInner { candidate_entries: HashMap, } +impl V1ReadBackend for TestStoreInner { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + self.load_candidate_entry(candidate_hash) + } + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult> { + self.load_block_entry(block_hash) + } +} + impl Backend for TestStoreInner { fn load_block_entry(&self, block_hash: &Hash) -> SubsystemResult> { Ok(self.block_entries.get(block_hash).cloned()) @@ -342,6 +358,18 @@ pub struct TestStore { store: Arc>, } +impl V1ReadBackend for TestStore { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + self.load_candidate_entry(candidate_hash) + } + fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult> { + self.load_block_entry(block_hash) + } +} + impl Backend for TestStore { fn load_block_entry(&self, block_hash: &Hash) -> SubsystemResult> { let store = self.store.lock(); @@ -391,6 +419,17 @@ fn garbage_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } } +fn garbage_assignment_cert_v2(kind: AssignmentCertKindV2) -> AssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCertV2 { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} + fn sign_approval( key: Sr25519Keyring, candidate_hash: CandidateHash, @@ -467,6 +506,12 @@ fn test_harness>( config: HarnessConfig, test: impl FnOnce(TestHarness) -> T, ) { + let _ = env_logger::builder() + .is_test(true) + .filter(Some("polkadot_node_core_approval_voting"), log::LevelFilter::Trace) + .filter(Some(LOG_TARGET), log::LevelFilter::Trace) + .try_init(); + let HarnessConfig { sync_oracle, sync_oracle_handle, clock, backend, assignment_criteria } = config; @@ -616,12 +661,13 @@ async fn check_and_import_assignment( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash, validator, - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), }, - candidate_index, + candidate_index.into(), tx, ), }, @@ -630,6 +676,38 @@ async fn check_and_import_assignment( rx } +async fn check_and_import_assignment_v2( + overseer: &mut VirtualOverseer, + block_hash: Hash, + core_indices: Vec, + validator: ValidatorIndex, +) -> oneshot::Receiver { + let (tx, rx) = oneshot::channel(); + overseer_send( + overseer, + FromOrchestra::Communication { + msg: ApprovalVotingMessage::CheckAndImportAssignment( + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: core_indices + .clone() + .into_iter() + .map(|c| CoreIndex(c)) + .collect::>() + .try_into() + .unwrap(), + }), + }, + core_indices.try_into().unwrap(), + tx, + ), + }, + ) + .await; + rx +} struct BlockConfig { slot: Slot, candidates: Option>, @@ -742,7 +820,7 @@ fn session_info(keys: &[Sr25519Keyring]) -> SessionInfo { vec![ValidatorIndex(0)], vec![ValidatorIndex(1)], ]), - n_cores: keys.len() as _, + n_cores: 10, needed_approvals: 2, zeroth_delay_tranche_width: 5, relay_vrf_modulo_samples: 3, @@ -1059,14 +1137,15 @@ fn blank_subsystem_act_on_bad_block() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: bad_block_hash, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -1322,9 +1401,22 @@ fn subsystem_accepts_duplicate_assignment() { } ); - let block_hash = Hash::repeat_byte(0x01); - let candidate_index = 0; let validator = ValidatorIndex(0); + let candidate_index = 0; + let block_hash = Hash::repeat_byte(0x01); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt + }; + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt + }; + let candidate_index1 = 0; + let candidate_index2 = 1; // Add block hash 00. ChainBuilder::new() @@ -1332,21 +1424,30 @@ fn subsystem_accepts_duplicate_assignment() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: Some(vec![ + (candidate_receipt1, CoreIndex(0), GroupIndex(1)), + (candidate_receipt2, CoreIndex(1), GroupIndex(1)), + ]), + session_info: None, + }, ) .build(&mut virtual_overseer) .await; - let rx = check_and_import_assignment( + // Initial assignment. + let rx = check_and_import_assignment_v2( &mut virtual_overseer, block_hash, - candidate_index, + vec![candidate_index1, candidate_index2], validator, ) .await; assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted)); + // Test with single assigned core. let rx = check_and_import_assignment( &mut virtual_overseer, block_hash, @@ -1357,6 +1458,18 @@ fn subsystem_accepts_duplicate_assignment() { assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate)); + // Test with multiple assigned cores. This cannot happen in practice, as tranche0 + // assignments are sent first, but we should still ensure correct behavior. + let rx = check_and_import_assignment_v2( + &mut virtual_overseer, + block_hash, + vec![candidate_index1, candidate_index2], + validator, + ) + .await; + + assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate)); + virtual_overseer }); } @@ -1407,6 +1520,63 @@ fn subsystem_rejects_assignment_with_unknown_candidate() { }); } +#[test] +fn subsystem_rejects_oversized_bitfields() { + test_harness(HarnessConfig::default(), |test_harness| async move { + let TestHarness { mut virtual_overseer, sync_oracle_handle: _sync_oracle_handle, .. } = + test_harness; + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let candidate_index = 10; + let validator = ValidatorIndex(0); + + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + ) + .build(&mut virtual_overseer) + .await; + + let rx = check_and_import_assignment( + &mut virtual_overseer, + block_hash, + candidate_index, + validator, + ) + .await; + + assert_eq!( + rx.await, + Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield( + candidate_index as usize + 1 + ))), + ); + + let rx = check_and_import_assignment_v2( + &mut virtual_overseer, + block_hash, + vec![1, 2, 10, 50], + validator, + ) + .await; + + assert_eq!( + rx.await, + Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidBitfield(51))), + ); + virtual_overseer + }); +} + #[test] fn subsystem_accepts_and_imports_approval_after_assignment() { test_harness(HarnessConfig::default(), |test_harness| async move { @@ -1727,14 +1897,15 @@ fn linear_import_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -1797,14 +1968,15 @@ fn forkful_import_at_same_height_act_on_leaf() { &mut virtual_overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportAssignment( - IndirectAssignmentCert { + IndirectAssignmentCertV2 { block_hash: head, validator: 0u32.into(), cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0, - }), + }) + .into(), }, - 0u32, + 0u32.into(), tx, ), }, @@ -2248,8 +2420,24 @@ fn subsystem_validate_approvals_cache() { let mut assignments = HashMap::new(); let _ = assignments.insert( CoreIndex(0), - approval_db::v1::OurAssignment { - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }), tranche: 0, validator_index: ValidatorIndex(0), triggered: false, @@ -2346,6 +2534,137 @@ fn subsystem_validate_approvals_cache() { }); } +#[test] +fn subsystem_doesnt_distribute_duplicate_compact_assignments() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let cert = garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1)].try_into().unwrap(), + }); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert, + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { + mut virtual_overseer, + sync_oracle_handle: _sync_oracle_handle, + clock, + .. + } = test_harness; + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt + }; + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt + }; + let candidate_index1 = 0; + let candidate_index2 = 1; + + // Add block hash 00. + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot: Slot::from(0), + candidates: Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(1)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]), + session_info: None, + }, + ) + .build(&mut virtual_overseer) + .await; + + // Activate the wakeup present above, and sleep to allow process_wakeups to execute.. + assert_eq!(Some(2), clock.inner.lock().next_wakeup()); + gum::trace!("clock \n{:?}\n", clock.inner.lock()); + + clock.inner.lock().wakeup_all(100); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(100)).await; + + // Assignment is distributed only once from `approval-voting` + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(c_indices, vec![candidate_index1, candidate_index2].try_into().unwrap()); + } + ); + + // Candidate 1 + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + // Candidate 2 + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + // Check if assignment was triggered for candidate 1. + let candidate_entry = + store.load_candidate_entry(&candidate_receipt1.hash()).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + // Check if assignment was triggered for candidate 2. + let candidate_entry = + store.load_candidate_entry(&candidate_receipt2.hash()).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + virtual_overseer + }); +} + /// Ensure that when two assignments are imported, only one triggers the Approval Checking work async fn handle_double_assignment_import( virtual_overseer: &mut VirtualOverseer, @@ -2355,9 +2674,9 @@ async fn handle_double_assignment_import( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( _, - c_index, + c_indices, )) => { - assert_eq!(candidate_index, c_index); + assert_eq!(Into::::into(candidate_index), c_indices); } ); @@ -2370,7 +2689,7 @@ async fn handle_double_assignment_import( _, c_index )) => { - assert_eq!(candidate_index, c_index); + assert_eq!(Into::::into(candidate_index), c_index); } ); @@ -2391,7 +2710,6 @@ async fn handle_double_assignment_import( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) ); - // Assert that there are no more messages being sent by the subsystem assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); } @@ -2466,8 +2784,9 @@ where let mut assignments = HashMap::new(); let _ = assignments.insert( CoreIndex(0), - approval_db::v1::OurAssignment { - cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), tranche: our_assigned_tranche, validator_index: ValidatorIndex(0), triggered: false, @@ -2601,14 +2920,12 @@ async fn step_until_done(clock: &MockClock) { futures_timer::Delay::new(Duration::from_millis(200)).await; let mut clock = clock.inner.lock(); if let Some(tick) = clock.next_wakeup() { - println!("TICK: {:?}", tick); relevant_ticks.push(tick); clock.set_tick(tick); } else { break } } - println!("relevant_ticks: {:?}", relevant_ticks); } #[test] diff --git a/polkadot/node/core/approval-voting/src/time.rs b/polkadot/node/core/approval-voting/src/time.rs index 34132dc22b23..a45866402c82 100644 --- a/polkadot/node/core/approval-voting/src/time.rs +++ b/polkadot/node/core/approval-voting/src/time.rs @@ -17,7 +17,7 @@ //! Time utilities for approval voting. use futures::prelude::*; -use polkadot_node_primitives::approval::DelayTranche; +use polkadot_node_primitives::approval::v1::DelayTranche; use sp_consensus_slots::Slot; use std::{ pin::Pin, diff --git a/polkadot/node/core/dispute-coordinator/src/import.rs b/polkadot/node/core/dispute-coordinator/src/import.rs index 0da3723ebf22..6222aab1cb10 100644 --- a/polkadot/node/core/dispute-coordinator/src/import.rs +++ b/polkadot/node/core/dispute-coordinator/src/import.rs @@ -109,7 +109,7 @@ pub enum OwnVoteState { } impl OwnVoteState { - fn new(votes: &CandidateVotes, env: &CandidateEnvironment<'_>) -> Self { + fn new(votes: &CandidateVotes, env: &CandidateEnvironment) -> Self { let controlled_indices = env.controlled_indices(); if controlled_indices.is_empty() { return Self::CannotVote diff --git a/polkadot/node/network/approval-distribution/Cargo.toml b/polkadot/node/network/approval-distribution/Cargo.toml index 9638c53d318b..15602a9796b5 100644 --- a/polkadot/node/network/approval-distribution/Cargo.toml +++ b/polkadot/node/network/approval-distribution/Cargo.toml @@ -14,10 +14,12 @@ polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-primitives = { path = "../../../primitives" } polkadot-node-jaeger = { path = "../../jaeger" } rand = "0.8" +itertools = "0.10.5" futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../../gum" } +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } [dev-dependencies] sp-authority-discovery = { path = "../../../../substrate/primitives/authority-discovery" } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index ac525ea6faf3..746a4b4dab5c 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -20,17 +20,23 @@ #![warn(missing_docs)] +use self::metrics::Metrics; use futures::{channel::oneshot, select, FutureExt as _}; +use itertools::Itertools; +use net_protocol::peer_set::{ProtocolVersion, ValidationVersion}; use polkadot_node_jaeger as jaeger; use polkadot_node_network_protocol::{ - self as net_protocol, + self as net_protocol, filter_by_peer_version, grid_topology::{RandomRouting, RequiredRouting, SessionGridTopologies, SessionGridTopology}, - peer_set::{ValidationVersion, MAX_NOTIFICATION_SIZE}, + peer_set::MAX_NOTIFICATION_SIZE, v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, UnifiedReputationChange as Rep, - Versioned, VersionedValidationProtocol, View, + Versioned, View, }; use polkadot_node_primitives::approval::{ - AssignmentCert, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, + v1::{ + AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, + }, + v2::{AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2}, }; use polkadot_node_subsystem::{ messages::{ @@ -49,8 +55,6 @@ use std::{ time::Duration, }; -use self::metrics::Metrics; - mod metrics; #[cfg(test)] @@ -64,11 +68,15 @@ const COST_DUPLICATE_MESSAGE: Rep = Rep::CostMinorRepeated("Peer sent identical const COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE: Rep = Rep::CostMinor("The vote was valid but too far in the future"); const COST_INVALID_MESSAGE: Rep = Rep::CostMajor("The vote was bad"); +const COST_OVERSIZED_BITFIELD: Rep = Rep::CostMajor("Oversized certificate or candidate bitfield"); const BENEFIT_VALID_MESSAGE: Rep = Rep::BenefitMinor("Peer sent a valid message"); const BENEFIT_VALID_MESSAGE_FIRST: Rep = Rep::BenefitMinorFirst("Valid message with new information"); +// Maximum valid size for the `CandidateBitfield` in the assignment messages. +const MAX_BITFIELD_SIZE: usize = 500; + /// The Approval Distribution subsystem. pub struct ApprovalDistribution { metrics: Metrics, @@ -97,7 +105,144 @@ impl RecentlyOutdated { } } -// In case the original gtid topology mechanisms don't work on their own, we need to trade bandwidth +// Contains topology routing information for assignments and approvals. +struct ApprovalRouting { + required_routing: RequiredRouting, + local: bool, + random_routing: RandomRouting, +} + +// This struct is responsible for tracking the full state of an assignment and grid routing +// information. +struct ApprovalEntry { + // The assignment certificate. + assignment: IndirectAssignmentCertV2, + // The candidates claimed by the certificate. A mapping between bit index and candidate index. + candidates: CandidateBitfield, + // The approval signatures for each `CandidateIndex` claimed by the assignment certificate. + approvals: HashMap, + // The validator index of the assignment signer. + validator_index: ValidatorIndex, + // Information required for gossiping to other peers using the grid topology. + routing_info: ApprovalRouting, +} + +#[derive(Debug)] +enum ApprovalEntryError { + InvalidValidatorIndex, + CandidateIndexOutOfBounds, + InvalidCandidateIndex, + DuplicateApproval, +} + +impl ApprovalEntry { + pub fn new( + assignment: IndirectAssignmentCertV2, + candidates: CandidateBitfield, + routing_info: ApprovalRouting, + ) -> ApprovalEntry { + Self { + validator_index: assignment.validator, + assignment, + approvals: HashMap::with_capacity(candidates.len()), + candidates, + routing_info, + } + } + + // Create a `MessageSubject` to reference the assignment. + pub fn create_assignment_knowledge(&self, block_hash: Hash) -> (MessageSubject, MessageKind) { + ( + MessageSubject(block_hash, self.candidates.clone(), self.validator_index), + MessageKind::Assignment, + ) + } + + // Create a `MessageSubject` to reference the approval. + pub fn create_approval_knowledge( + &self, + block_hash: Hash, + candidate_index: CandidateIndex, + ) -> (MessageSubject, MessageKind) { + ( + MessageSubject(block_hash, candidate_index.into(), self.validator_index), + MessageKind::Approval, + ) + } + + // Updates routing information and returns the previous information if any. + pub fn routing_info_mut(&mut self) -> &mut ApprovalRouting { + &mut self.routing_info + } + + // Get the routing information. + pub fn routing_info(&self) -> &ApprovalRouting { + &self.routing_info + } + + // Update routing information. + pub fn update_required_routing(&mut self, required_routing: RequiredRouting) { + self.routing_info.required_routing = required_routing; + } + + // Records a new approval. Returns false if the claimed candidate is not found or we already + // have received the approval. + pub fn note_approval( + &mut self, + approval: IndirectSignedApprovalVote, + ) -> Result<(), ApprovalEntryError> { + // First do some sanity checks: + // - check validator index matches + // - check claimed candidate + // - check for duplicate approval + if self.validator_index != approval.validator { + return Err(ApprovalEntryError::InvalidValidatorIndex) + } + + if self.candidates.len() <= approval.candidate_index as usize { + return Err(ApprovalEntryError::CandidateIndexOutOfBounds) + } + + if !self.candidates.bit_at(approval.candidate_index.as_bit_index()) { + return Err(ApprovalEntryError::InvalidCandidateIndex) + } + + if self.approvals.contains_key(&approval.candidate_index) { + return Err(ApprovalEntryError::DuplicateApproval) + } + + self.approvals.insert(approval.candidate_index, approval); + Ok(()) + } + + // Get the assignment certiticate and claimed candidates. + pub fn assignment(&self) -> (IndirectAssignmentCertV2, CandidateBitfield) { + (self.assignment.clone(), self.candidates.clone()) + } + + // Get all approvals for all candidates claimed by the assignment. + pub fn approvals(&self) -> Vec { + self.approvals.values().cloned().collect::>() + } + + // Get the approval for a specific candidate index. + pub fn approval(&self, candidate_index: CandidateIndex) -> Option { + self.approvals.get(&candidate_index).cloned() + } + + // Get validator index. + pub fn validator_index(&self) -> ValidatorIndex { + self.validator_index + } +} + +// We keep track of each peer view and protocol version using this struct. +struct PeerEntry { + pub view: View, + pub version: ProtocolVersion, +} + +// In case the original grid topology mechanisms don't work on their own, we need to trade bandwidth // for protocol liveliness by introducing aggression. // // Aggression has 3 levels: @@ -117,7 +262,6 @@ impl RecentlyOutdated { // not to all blocks older than the threshold. Most likely, a few assignments struggle to // be propagated in a single block and this holds up all of its descendants blocks. // Accordingly, we only step on the gas for the block which is most obviously holding up finality. - /// Aggression configuration representation #[derive(Clone)] struct AggressionConfig { @@ -160,15 +304,6 @@ enum Resend { No, } -/// Data stored on a per-peer basis. -#[derive(Debug)] -struct PeerData { - /// The peer's view. - view: View, - /// The peer's protocol version. - version: ValidationVersion, -} - /// The [`State`] struct is responsible for tracking the overall state of the subsystem. /// /// It tracks metadata about our view of the unfinalized chain, @@ -189,7 +324,7 @@ struct State { pending_known: HashMap>, /// Peer data is partially stored here, and partially inline within the [`BlockEntry`]s - peer_data: HashMap, + peer_views: HashMap, /// Keeps a topology for various different sessions. topologies: SessionGridTopologies, @@ -197,12 +332,12 @@ struct State { /// Tracks recently finalized blocks. recent_outdated_blocks: RecentlyOutdated, - /// Config for aggression. - aggression_config: AggressionConfig, - - /// `HashMap` from active leaves to spans + /// HashMap from active leaves to spans spans: HashMap, + /// Aggression configuration. + aggression_config: AggressionConfig, + /// Current approval checking finality lag. approval_checking_lag: BlockNumber, @@ -216,8 +351,11 @@ enum MessageKind { Approval, } +// Utility structure to identify assignments and approvals for specific candidates. +// Assignments can span multiple candidates, while approvals refer to only one candidate. +// #[derive(Debug, Clone, Hash, PartialEq, Eq)] -struct MessageSubject(Hash, CandidateIndex, ValidatorIndex); +struct MessageSubject(Hash, pub CandidateBitfield, ValidatorIndex); #[derive(Debug, Clone, Default)] struct Knowledge { @@ -238,9 +376,11 @@ impl Knowledge { } fn insert(&mut self, message: MessageSubject, kind: MessageKind) -> bool { - match self.known_messages.entry(message) { + let mut success = match self.known_messages.entry(message.clone()) { hash_map::Entry::Vacant(vacant) => { vacant.insert(kind); + // If there are multiple candidates assigned in the message, create + // separate entries for each one. true }, hash_map::Entry::Occupied(mut occupied) => match (*occupied.get(), kind) { @@ -252,7 +392,25 @@ impl Knowledge { true }, }, + }; + + // In case of succesful insertion of multiple candidate assignments create additional + // entries for each assigned candidate. This fakes knowledge of individual assignments, but + // we need to share the same `MessageSubject` with the followup approval candidate index. + if kind == MessageKind::Assignment && success && message.1.count_ones() > 1 { + for candidate_index in message.1.iter_ones() { + success = success && + self.insert( + MessageSubject( + message.0, + vec![candidate_index as u32].try_into().expect("Non-empty vec; qed"), + message.2, + ), + kind, + ); + } } + success } } @@ -286,47 +444,97 @@ struct BlockEntry { candidates: Vec, /// The session index of this block. session: SessionIndex, + /// Approval entries for whole block. These also contain all approvals in the case of multiple + /// candidates being claimed by assignments. + approval_entries: HashMap<(ValidatorIndex, CandidateBitfield), ApprovalEntry>, } -#[derive(Debug)] -enum ApprovalState { - Assigned(AssignmentCert), - Approved(AssignmentCert, ValidatorSignature), -} +impl BlockEntry { + // Returns the peer which currently know this block. + pub fn known_by(&self) -> Vec { + self.known_by.keys().cloned().collect::>() + } -impl ApprovalState { - fn assignment_cert(&self) -> &AssignmentCert { - match *self { - ApprovalState::Assigned(ref cert) => cert, - ApprovalState::Approved(ref cert, _) => cert, + pub fn insert_approval_entry(&mut self, entry: ApprovalEntry) -> &mut ApprovalEntry { + // First map one entry per candidate to the same key we will use in `approval_entries`. + // Key is (Validator_index, CandidateBitfield) that links the `ApprovalEntry` to the (K,V) + // entry in `candidate_entry.messages`. + for claimed_candidate_index in entry.candidates.iter_ones() { + match self.candidates.get_mut(claimed_candidate_index) { + Some(candidate_entry) => { + candidate_entry + .messages + .entry(entry.validator_index()) + .or_insert(entry.candidates.clone()); + }, + None => { + // This should never happen, but if it happens, it means the subsystem is + // broken. + gum::warn!( + target: LOG_TARGET, + hash = ?entry.assignment.block_hash, + ?claimed_candidate_index, + "Missing candidate entry on `import_and_circulate_assignment`", + ); + }, + }; } + + self.approval_entries + .entry((entry.validator_index, entry.candidates.clone())) + .or_insert(entry) } - fn approval_signature(&self) -> Option { - match *self { - ApprovalState::Assigned(_) => None, - ApprovalState::Approved(_, ref sig) => Some(sig.clone()), - } + // Returns a mutable reference of `ApprovalEntry` for `candidate_index` from validator + // `validator_index`. + pub fn approval_entry( + &mut self, + candidate_index: CandidateIndex, + validator_index: ValidatorIndex, + ) -> Option<&mut ApprovalEntry> { + self.candidates + .get(candidate_index as usize) + .map_or(None, |candidate_entry| candidate_entry.messages.get(&validator_index)) + .map_or(None, |candidate_indices| { + self.approval_entries.get_mut(&(validator_index, candidate_indices.clone())) + }) } -} -// routing state bundled with messages for the candidate. Corresponding assignments -// and approvals are stored together and should be routed in the same way, with -// assignments preceding approvals in all cases. -#[derive(Debug)] -struct MessageState { - required_routing: RequiredRouting, - local: bool, - random_routing: RandomRouting, - approval_state: ApprovalState, + // Get all approval entries for a given candidate. + pub fn approval_entries(&self, candidate_index: CandidateIndex) -> Vec<&ApprovalEntry> { + // Get the keys for fetching `ApprovalEntry` from `self.approval_entries`, + let approval_entry_keys = self + .candidates + .get(candidate_index as usize) + .map(|candidate_entry| &candidate_entry.messages); + + if let Some(approval_entry_keys) = approval_entry_keys { + // Ensure no duplicates. + let approval_entry_keys = approval_entry_keys.iter().unique().collect::>(); + + let mut entries = Vec::new(); + for (validator_index, candidate_indices) in approval_entry_keys { + if let Some(entry) = + self.approval_entries.get(&(*validator_index, candidate_indices.clone())) + { + entries.push(entry); + } + } + entries + } else { + vec![] + } + } } -/// Information about candidates in the context of a particular block they are included in. -/// In other words, multiple `CandidateEntry`s may exist for the same candidate, -/// if it is included by multiple blocks - this is likely the case when there are forks. +// Information about candidates in the context of a particular block they are included in. +// In other words, multiple `CandidateEntry`s may exist for the same candidate, +// if it is included by multiple blocks - this is likely the case when there are forks. #[derive(Debug, Default)] struct CandidateEntry { - messages: HashMap, + // The value represents part of the lookup key in `approval_entries` to fetch the assignment + // and existing votes. + messages: HashMap, } #[derive(Debug, Clone, PartialEq)] @@ -345,7 +553,7 @@ impl MessageSource { } enum PendingMessage { - Assignment(IndirectAssignmentCert, CandidateIndex), + Assignment(IndirectAssignmentCertV2, CandidateBitfield), Approval(IndirectSignedApprovalVote), } @@ -362,27 +570,13 @@ impl State { NetworkBridgeEvent::PeerConnected(peer_id, role, version, _) => { // insert a blank view if none already present gum::trace!(target: LOG_TARGET, ?peer_id, ?role, "Peer connected"); - let version = match ValidationVersion::try_from(version).ok() { - Some(v) => v, - None => { - // sanity: network bridge is supposed to detect this already. - gum::error!( - target: LOG_TARGET, - ?peer_id, - ?version, - "Unsupported protocol version" - ); - return - }, - }; - - self.peer_data + self.peer_views .entry(peer_id) - .or_insert_with(|| PeerData { version, view: Default::default() }); + .or_insert(PeerEntry { view: Default::default(), version }); }, NetworkBridgeEvent::PeerDisconnected(peer_id) => { gum::trace!(target: LOG_TARGET, ?peer_id, "Peer disconnected"); - self.peer_data.remove(&peer_id); + self.peer_views.remove(&peer_id); self.blocks.iter_mut().for_each(|(_hash, entry)| { entry.known_by.remove(&peer_id); }) @@ -419,8 +613,8 @@ impl State { live }); }, - NetworkBridgeEvent::PeerMessage(peer_id, msg) => { - self.process_incoming_peer_message(ctx, metrics, peer_id, msg, rng).await; + NetworkBridgeEvent::PeerMessage(peer_id, message) => { + self.process_incoming_peer_message(ctx, metrics, peer_id, message, rng).await; }, NetworkBridgeEvent::UpdatedAuthorityIds { .. } => { // The approval-distribution subsystem doesn't deal with `AuthorityDiscoveryId`s. @@ -459,6 +653,7 @@ impl State { knowledge: Knowledge::default(), candidates, session: meta.session, + approval_entries: HashMap::new(), }); self.topologies.inc_session_refs(meta.session); @@ -481,18 +676,17 @@ impl State { { let sender = ctx.sender(); - for (peer_id, data) in self.peer_data.iter() { - let intersection = data.view.iter().filter(|h| new_hashes.contains(h)); - let view_intersection = - View::new(intersection.cloned(), data.view.finalized_number); + for (peer_id, PeerEntry { view, version }) in self.peer_views.iter() { + let intersection = view.iter().filter(|h| new_hashes.contains(h)); + let view_intersection = View::new(intersection.cloned(), view.finalized_number); Self::unify_with_peer( sender, metrics, &mut self.blocks, &self.topologies, - self.peer_data.len(), + self.peer_views.len(), *peer_id, - data.version, + *version, view_intersection, rng, ) @@ -530,13 +724,13 @@ impl State { for (peer_id, message) in to_import { match message { - PendingMessage::Assignment(assignment, claimed_index) => { + PendingMessage::Assignment(assignment, claimed_indices) => { self.import_and_circulate_assignment( ctx, metrics, MessageSource::Peer(peer_id), assignment, - claimed_index, + claimed_indices, rng, ) .await; @@ -575,33 +769,78 @@ impl State { adjust_required_routing_and_propagate( ctx, - &self.peer_data, &mut self.blocks, &self.topologies, |block_entry| block_entry.session == session, |required_routing, local, validator_index| { - if *required_routing == RequiredRouting::PendingTopology { - *required_routing = topology + if required_routing == &RequiredRouting::PendingTopology { + topology .local_grid_neighbors() - .required_routing_by_index(*validator_index, local); + .required_routing_by_index(*validator_index, local) + } else { + *required_routing } }, + &self.peer_views, ) .await; } + async fn process_incoming_assignments( + &mut self, + ctx: &mut Context, + metrics: &Metrics, + peer_id: PeerId, + assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + rng: &mut R, + ) where + R: CryptoRng + Rng, + { + for (assignment, claimed_indices) in assignments { + if let Some(pending) = self.pending_known.get_mut(&assignment.block_hash) { + let block_hash = &assignment.block_hash; + let validator_index = assignment.validator; + + gum::trace!( + target: LOG_TARGET, + %peer_id, + ?block_hash, + ?claimed_indices, + ?validator_index, + "Pending assignment", + ); + + pending.push((peer_id, PendingMessage::Assignment(assignment, claimed_indices))); + + continue + } + + self.import_and_circulate_assignment( + ctx, + metrics, + MessageSource::Peer(peer_id), + assignment, + claimed_indices, + rng, + ) + .await; + } + } + async fn process_incoming_peer_message( &mut self, ctx: &mut Context, metrics: &Metrics, peer_id: PeerId, - msg: net_protocol::ApprovalDistributionMessage, + msg: Versioned< + protocol_v1::ApprovalDistributionMessage, + protocol_vstaging::ApprovalDistributionMessage, + >, rng: &mut R, ) where R: CryptoRng + Rng, { match msg { - Versioned::V1(protocol_v1::ApprovalDistributionMessage::Assignments(assignments)) | Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Assignments( assignments, )) => { @@ -611,42 +850,42 @@ impl State { num = assignments.len(), "Processing assignments from a peer", ); - for (assignment, claimed_index) in assignments.into_iter() { - if let Some(pending) = self.pending_known.get_mut(&assignment.block_hash) { - let message_subject = MessageSubject( - assignment.block_hash, - claimed_index, - assignment.validator, - ); - - gum::trace!( - target: LOG_TARGET, - %peer_id, - ?message_subject, - "Pending assignment", - ); + let sanitized_assignments = + self.sanitize_v2_assignments(peer_id, ctx.sender(), assignments).await; - pending - .push((peer_id, PendingMessage::Assignment(assignment, claimed_index))); + self.process_incoming_assignments( + ctx, + metrics, + peer_id, + sanitized_assignments, + rng, + ) + .await; + }, + Versioned::V1(protocol_v1::ApprovalDistributionMessage::Assignments(assignments)) => { + gum::trace!( + target: LOG_TARGET, + peer_id = %peer_id, + num = assignments.len(), + "Processing assignments from a peer", + ); - continue - } + let sanitized_assignments = + self.sanitize_v1_assignments(peer_id, ctx.sender(), assignments).await; - self.import_and_circulate_assignment( - ctx, - metrics, - MessageSource::Peer(peer_id), - assignment, - claimed_index, - rng, - ) - .await; - } + self.process_incoming_assignments( + ctx, + metrics, + peer_id, + sanitized_assignments, + rng, + ) + .await; }, - Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) | Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Approvals( approvals, - )) => { + )) | + Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) => { gum::trace!( target: LOG_TARGET, peer_id = %peer_id, @@ -655,17 +894,17 @@ impl State { ); for approval_vote in approvals.into_iter() { if let Some(pending) = self.pending_known.get_mut(&approval_vote.block_hash) { - let message_subject = MessageSubject( - approval_vote.block_hash, - approval_vote.candidate_index, - approval_vote.validator, - ); + let block_hash = approval_vote.block_hash; + let candidate_index = approval_vote.candidate_index; + let validator_index = approval_vote.validator; gum::trace!( target: LOG_TARGET, %peer_id, - ?message_subject, - "Pending approval", + ?block_hash, + ?candidate_index, + ?validator_index, + "Pending assignment", ); pending.push((peer_id, PendingMessage::Approval(approval_vote))); @@ -699,14 +938,23 @@ impl State { { gum::trace!(target: LOG_TARGET, ?view, "Peer view change"); let finalized_number = view.finalized_number; - let (peer_protocol_version, old_finalized_number) = match self - .peer_data - .get_mut(&peer_id) - .map(|d| (d.version, std::mem::replace(&mut d.view, view.clone()))) - { - Some((v, view)) => (v, view.finalized_number), - None => return, // unknown peer - }; + + let (old_view, protocol_version) = + if let Some(peer_entry) = self.peer_views.get_mut(&peer_id) { + (Some(std::mem::replace(&mut peer_entry.view, view.clone())), peer_entry.version) + } else { + // This shouldn't happen, but if it does we assume protocol version 1. + gum::warn!( + target: LOG_TARGET, + ?peer_id, + ?view, + "Peer view change for missing `peer_entry`" + ); + + (None, ValidationVersion::V1.into()) + }; + + let old_finalized_number = old_view.map(|v| v.finalized_number).unwrap_or(0); // we want to prune every block known_by peer up to (including) view.finalized_number let blocks = &mut self.blocks; @@ -731,9 +979,9 @@ impl State { metrics, &mut self.blocks, &self.topologies, - self.peer_data.len(), + self.peer_views.len(), peer_id, - peer_protocol_version, + protocol_version, view, rng, ) @@ -774,8 +1022,8 @@ impl State { ctx: &mut Context, metrics: &Metrics, source: MessageSource, - assignment: IndirectAssignmentCert, - claimed_candidate_index: CandidateIndex, + assignment: IndirectAssignmentCertV2, + claimed_candidate_indices: CandidateBitfield, rng: &mut R, ) where R: CryptoRng + Rng, @@ -823,9 +1071,11 @@ impl State { }, }; - // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, claimed_candidate_index, validator_index); - let message_kind = MessageKind::Assignment; + // Compute metadata on the assignment. + let (message_subject, message_kind) = ( + MessageSubject(block_hash, claimed_candidate_indices.clone(), validator_index), + MessageKind::Assignment, + ); if let Some(peer_id) = source.peer_id() { // check if our knowledge of the peer already contains this assignment @@ -841,6 +1091,7 @@ impl State { ?message_subject, "Duplicate assignment", ); + modify_reputation( &mut self.reputation, ctx.sender(), @@ -848,6 +1099,15 @@ impl State { COST_DUPLICATE_MESSAGE, ) .await; + } else { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + hash = ?block_hash, + ?validator_index, + ?message_subject, + "We sent the message to the peer while peer was sending it to us. Known race condition.", + ); } return } @@ -889,7 +1149,7 @@ impl State { ctx.send_message(ApprovalVotingMessage::CheckAndImportAssignment( assignment.clone(), - claimed_candidate_index, + claimed_candidate_indices.clone(), tx, )) .await; @@ -920,7 +1180,7 @@ impl State { BENEFIT_VALID_MESSAGE_FIRST, ) .await; - entry.knowledge.known_messages.insert(message_subject.clone(), message_kind); + entry.knowledge.insert(message_subject.clone(), message_kind); if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { peer_knowledge.received.insert(message_subject.clone(), message_kind); } @@ -994,7 +1254,7 @@ impl State { // Invariant: to our knowledge, none of the peers except for the `source` know about the // assignment. - metrics.on_assignment_imported(); + metrics.on_assignment_imported(&assignment.cert.kind); let topology = self.topologies.get_topology(entry.session); let local = source == MessageSource::Local; @@ -1003,28 +1263,14 @@ impl State { t.local_grid_neighbors().required_routing_by_index(validator_index, local) }); - let message_state = match entry.candidates.get_mut(claimed_candidate_index as usize) { - Some(candidate_entry) => { - // set the approval state for validator_index to Assigned - // unless the approval state is set already - candidate_entry.messages.entry(validator_index).or_insert_with(|| MessageState { - required_routing, - local, - random_routing: Default::default(), - approval_state: ApprovalState::Assigned(assignment.cert.clone()), - }) - }, - None => { - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?claimed_candidate_index, - "Expected a candidate entry on import_and_circulate_assignment", - ); + // All the peers that know the relay chain block. + let peers_to_filter = entry.known_by(); - return - }, - }; + let approval_entry = entry.insert_approval_entry(ApprovalEntry::new( + assignment.clone(), + claimed_candidate_indices.clone(), + ApprovalRouting { required_routing, local, random_routing: Default::default() }, + )); // Dispatch the message to all peers in the routing set which // know the block. @@ -1032,83 +1278,65 @@ impl State { // If the topology isn't known yet (race with networking subsystems) // then messages will be sent when we get it. - let assignments = vec![(assignment, claimed_candidate_index)]; - let n_peers_total = self.peer_data.len(); + let assignments = vec![(assignment, claimed_candidate_indices.clone())]; + let n_peers_total = self.peer_views.len(); let source_peer = source.peer_id(); - let mut peer_filter = move |peer| { - if Some(peer) == source_peer.as_ref() { - return false + // Peers that we will send the assignment to. + let mut peers = Vec::new(); + + // Filter destination peers + for peer in peers_to_filter.into_iter() { + if Some(peer) == source_peer { + continue } if let Some(true) = topology .as_ref() - .map(|t| t.local_grid_neighbors().route_to_peer(required_routing, peer)) + .map(|t| t.local_grid_neighbors().route_to_peer(required_routing, &peer)) { - return true + peers.push(peer); + continue } // Note: at this point, we haven't received the message from any peers // other than the source peer, and we just got it, so we haven't sent it // to any peers either. - let route_random = message_state.random_routing.sample(n_peers_total, rng); + let route_random = + approval_entry.routing_info().random_routing.sample(n_peers_total, rng); if route_random { - message_state.random_routing.inc_sent(); - } - - route_random - }; - - let (v1_peers, vstaging_peers) = { - let peer_data = &self.peer_data; - let peers = entry - .known_by - .keys() - .filter_map(|p| peer_data.get_key_value(p)) - .filter(|(p, _)| peer_filter(p)) - .map(|(p, peer_data)| (*p, peer_data.version)) - .collect::>(); - - // Add the metadata of the assignment to the knowledge of each peer. - for (peer, _) in peers.iter() { - // we already filtered peers above, so this should always be Some - if let Some(peer_knowledge) = entry.known_by.get_mut(peer) { - peer_knowledge.sent.insert(message_subject.clone(), message_kind); - } + approval_entry.routing_info_mut().random_routing.inc_sent(); + peers.push(peer); } + } - if !peers.is_empty() { - gum::trace!( - target: LOG_TARGET, - ?block_hash, - ?claimed_candidate_index, - local = source.peer_id().is_none(), - num_peers = peers.len(), - "Sending an assignment to peers", - ); + // Add the metadata of the assignment to the knowledge of each peer. + for peer in peers.iter() { + // we already filtered peers above, so this should always be Some + if let Some(peer_knowledge) = entry.known_by.get_mut(peer) { + peer_knowledge.sent.insert(message_subject.clone(), message_kind); } + } - let v1_peers = filter_peers_by_version(&peers, ValidationVersion::V1); - let vstaging_peers = filter_peers_by_version(&peers, ValidationVersion::VStaging); + if !peers.is_empty() { + gum::trace!( + target: LOG_TARGET, + ?block_hash, + ?claimed_candidate_indices, + local = source.peer_id().is_none(), + num_peers = peers.len(), + "Sending an assignment to peers", + ); - (v1_peers, vstaging_peers) - }; + let peers = peers + .iter() + .filter_map(|peer_id| { + self.peer_views.get(peer_id).map(|peer_entry| (*peer_id, peer_entry.version)) + }) + .collect::>(); - if !v1_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v1_peers, - versioned_assignments_packet(ValidationVersion::V1, assignments.clone()), - )) - .await; - } - - if !vstaging_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_peers, - versioned_assignments_packet(ValidationVersion::VStaging, assignments.clone()), - )) - .await; + send_assignments_batched(ctx.sender(), assignments, &peers).await; } } @@ -1143,6 +1371,14 @@ impl State { _ => { if let Some(peer_id) = source.peer_id() { if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { + gum::debug!( + target: LOG_TARGET, + ?peer_id, + ?block_hash, + ?validator_index, + ?candidate_index, + "Approval from a peer is out of view", + ); modify_reputation( &mut self.reputation, ctx.sender(), @@ -1157,7 +1393,7 @@ impl State { }; // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, candidate_index, validator_index); + let message_subject = MessageSubject(block_hash, candidate_index.into(), validator_index); let message_kind = MessageKind::Approval; if let Some(peer_id) = source.peer_id() { @@ -1307,71 +1543,48 @@ impl State { } } - // Invariant: to our knowledge, none of the peers except for the `source` know about the - // approval. - metrics.on_approval_imported(); - - let required_routing = match entry.candidates.get_mut(candidate_index as usize) { - Some(candidate_entry) => { - // set the approval state for validator_index to Approved - // it should be in assigned state already - match candidate_entry.messages.remove(&validator_index) { - Some(MessageState { - approval_state: ApprovalState::Assigned(cert), - required_routing, - local, - random_routing, - }) => { - candidate_entry.messages.insert( - validator_index, - MessageState { - approval_state: ApprovalState::Approved( - cert, - vote.signature.clone(), - ), - required_routing, - local, - random_routing, - }, - ); - - required_routing - }, - Some(_) => { - unreachable!( - "we only insert it after the metadata, checked the metadata above; qed" - ); - }, - None => { - // this would indicate a bug in approval-voting - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?candidate_index, - ?validator_index, - "Importing an approval we don't have an assignment for", - ); + let required_routing = match entry.approval_entry(candidate_index, validator_index) { + Some(approval_entry) => { + // Invariant: to our knowledge, none of the peers except for the `source` know about + // the approval. + metrics.on_approval_imported(); + + if let Err(err) = approval_entry.note_approval(vote.clone()) { + // this would indicate a bug in approval-voting: + // - validator index mismatch + // - candidate index mismatch + // - duplicate approval + gum::warn!( + target: LOG_TARGET, + hash = ?block_hash, + ?candidate_index, + ?validator_index, + ?err, + "Possible bug: Vote import failed", + ); - return - }, + return } + + approval_entry.routing_info().required_routing }, None => { + let peer_id = source.peer_id(); + // This indicates a bug in approval-distribution, since we check the knowledge at + // the begining of the function. gum::warn!( target: LOG_TARGET, - hash = ?block_hash, - ?candidate_index, - ?validator_index, - "Expected a candidate entry on import_and_circulate_approval", + ?peer_id, + ?message_subject, + "Unknown approval assignment", ); - + // No rep change as this is caused by an issue return }, }; // Dispatch a ApprovalDistributionV1Message::Approval(vote) // to all peers required by the topology, with the exception of the source peer. - let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); @@ -1395,57 +1608,56 @@ impl State { in_topology || knowledge.sent.contains(message_subject, MessageKind::Assignment) }; - let (v1_peers, vstaging_peers) = { - let peer_data = &self.peer_data; - let peers = entry - .known_by - .iter() - .filter_map(|(p, k)| peer_data.get(&p).map(|pd| (p, k, pd.version))) - .filter(|(p, k, _)| peer_filter(p, k)) - .map(|(p, _, v)| (*p, v)) - .collect::>(); - - // Add the metadata of the assignment to the knowledge of each peer. - for (peer, _) in peers.iter() { - // we already filtered peers above, so this should always be Some - if let Some(peer_knowledge) = entry.known_by.get_mut(peer) { - peer_knowledge.sent.insert(message_subject.clone(), message_kind); - } + let peers = entry + .known_by + .iter() + .filter(|(p, k)| peer_filter(p, k)) + .filter_map(|(p, _)| self.peer_views.get(p).map(|entry| (*p, entry.version))) + .collect::>(); + + // Add the metadata of the assignment to the knowledge of each peer. + for peer in peers.iter() { + // we already filtered peers above, so this should always be Some + if let Some(entry) = entry.known_by.get_mut(&peer.0) { + entry.sent.insert(message_subject.clone(), message_kind); } + } - if !peers.is_empty() { - gum::trace!( - target: LOG_TARGET, - ?block_hash, - ?candidate_index, - local = source.peer_id().is_none(), - num_peers = peers.len(), - "Sending an approval to peers", - ); - } - - let v1_peers = filter_peers_by_version(&peers, ValidationVersion::V1); - let vstaging_peers = filter_peers_by_version(&peers, ValidationVersion::VStaging); - - (v1_peers, vstaging_peers) - }; + if !peers.is_empty() { + let approvals = vec![vote]; + gum::trace!( + target: LOG_TARGET, + ?block_hash, + ?candidate_index, + local = source.peer_id().is_none(), + num_peers = peers.len(), + "Sending an approval to peers", + ); - let approvals = vec![vote]; + let v1_peers = filter_by_peer_version(&peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(&peers, ValidationVersion::VStaging.into()); - if !v1_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v1_peers, - versioned_approvals_packet(ValidationVersion::V1, approvals.clone()), - )) - .await; - } + if !v1_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v1_peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(approvals.clone()), + )), + )) + .await; + } - if !vstaging_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_peers, - versioned_approvals_packet(ValidationVersion::VStaging, approvals), - )) - .await; + if !v2_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + v2_peers, + Versioned::VStaging( + protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals), + ), + ), + )) + .await; + } } } @@ -1476,25 +1688,12 @@ impl State { Some(e) => e, }; - let candidate_entry = match block_entry.candidates.get(index as usize) { - None => { - gum::debug!( - target: LOG_TARGET, - ?hash, - ?index, - "`get_approval_signatures`: could not find candidate entry for given hash and index!" - ); - continue - }, - Some(e) => e, - }; - let sigs = - candidate_entry.messages.iter().filter_map(|(validator_index, message_state)| { - match &message_state.approval_state { - ApprovalState::Approved(_, sig) => Some((*validator_index, sig.clone())), - ApprovalState::Assigned(_) => None, - } - }); + let sigs = block_entry + .approval_entries(index) + .into_iter() + .filter_map(|approval_entry| approval_entry.approval(index)) + .map(|approval| (approval.validator, approval.signature)) + .collect::>(); all_sigs.extend(sigs); } all_sigs @@ -1507,7 +1706,7 @@ impl State { topologies: &SessionGridTopologies, total_peers: usize, peer_id: PeerId, - peer_protocol_version: ValidationVersion, + protocol_version: ProtocolVersion, view: View, rng: &mut (impl CryptoRng + Rng), ) { @@ -1520,6 +1719,8 @@ impl State { let view_finalized_number = view.finalized_number; for head in view.into_iter() { let mut block = head; + + // Walk the chain back to last finalized block of the peer view. loop { let entry = match entries.get_mut(&block) { Some(entry) if entry.number > view_finalized_number => entry, @@ -1534,19 +1735,16 @@ impl State { } let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let topology = topologies.get_topology(entry.session); - // Iterate all messages in all candidates. - for (candidate_index, validator, message_state) in - entry.candidates.iter_mut().enumerate().flat_map(|(c_i, c)| { - c.messages.iter_mut().map(move |(k, v)| (c_i as _, k, v)) - }) { + // We want to iterate the `approval_entries` of the block entry as these contain all + // assignments that also link all approval votes. + for approval_entry in entry.approval_entries.values_mut() { // Propagate the message to all peers in the required routing set OR // randomly sample peers. { - let random_routing = &mut message_state.random_routing; - let required_routing = message_state.required_routing; + let required_routing = approval_entry.routing_info().required_routing; + let random_routing = &mut approval_entry.routing_info_mut().random_routing; let rng = &mut *rng; let mut peer_filter = move |peer_id| { let in_topology = topology.as_ref().map_or(false, |t| { @@ -1567,39 +1765,24 @@ impl State { } } - let message_subject = MessageSubject(block, candidate_index, *validator); - - let assignment_message = ( - IndirectAssignmentCert { - block_hash: block, - validator: *validator, - cert: message_state.approval_state.assignment_cert().clone(), - }, - candidate_index, - ); - - let approval_message = - message_state.approval_state.approval_signature().map(|signature| { - IndirectSignedApprovalVote { - block_hash: block, - validator: *validator, - candidate_index, - signature, - } - }); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(block); - if !peer_knowledge.contains(&message_subject, MessageKind::Assignment) { - peer_knowledge - .sent - .insert(message_subject.clone(), MessageKind::Assignment); + // Only send stuff a peer doesn't know in the context of a relay chain block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge, message_kind); assignments_to_send.push(assignment_message); } - if let Some(approval_message) = approval_message { - if !peer_knowledge.contains(&message_subject, MessageKind::Approval) { - peer_knowledge - .sent - .insert(message_subject.clone(), MessageKind::Approval); + // Filter approval votes. + for approval_message in approval_messages { + let (approval_knowledge, message_kind) = approval_entry + .create_approval_knowledge(block, approval_message.candidate_index); + + if !peer_knowledge.contains(&approval_knowledge, message_kind) { + peer_knowledge.sent.insert(approval_knowledge, message_kind); approvals_to_send.push(approval_message); } } @@ -1613,23 +1796,30 @@ impl State { gum::trace!( target: LOG_TARGET, ?peer_id, + ?protocol_version, num = assignments_to_send.len(), "Sending assignments to unified peer", ); - send_assignments_batched(sender, assignments_to_send, peer_id, peer_protocol_version) - .await; + send_assignments_batched( + sender, + assignments_to_send, + &vec![(peer_id, protocol_version)], + ) + .await; } if !approvals_to_send.is_empty() { gum::trace!( target: LOG_TARGET, ?peer_id, + ?protocol_version, num = approvals_to_send.len(), "Sending approvals to unified peer", ); - send_approvals_batched(sender, approvals_to_send, peer_id, peer_protocol_version).await; + send_approvals_batched(sender, approvals_to_send, &vec![(peer_id, protocol_version)]) + .await; } } @@ -1665,7 +1855,6 @@ impl State { adjust_required_routing_and_propagate( ctx, - &self.peer_data, &mut self.blocks, &self.topologies, |block_entry| { @@ -1687,13 +1876,13 @@ impl State { false } }, - |_, _, _| {}, + |required_routing, _, _| *required_routing, + &self.peer_views, ) .await; adjust_required_routing_and_propagate( ctx, - &self.peer_data, &mut self.blocks, &self.topologies, |block_entry| { @@ -1712,30 +1901,110 @@ impl State { lag = ?self.approval_checking_lag, "Encountered old block pending gossip topology", ); - return + return *required_routing } + let mut new_required_routing = *required_routing; + if config.l1_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) { // Message originator sends to everyone. - if local && *required_routing != RequiredRouting::All { + if local && new_required_routing != RequiredRouting::All { metrics.on_aggression_l1(); - *required_routing = RequiredRouting::All; + new_required_routing = RequiredRouting::All; } } if config.l2_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) { // Message originator sends to everyone. Everyone else sends to XY. - if !local && *required_routing != RequiredRouting::GridXY { + if !local && new_required_routing != RequiredRouting::GridXY { metrics.on_aggression_l2(); - *required_routing = RequiredRouting::GridXY; + new_required_routing = RequiredRouting::GridXY; } } + new_required_routing }, + &self.peer_views, ) .await; } + + // Filter out invalid candidate index and certificate core bitfields. + // For each invalid assignment we also punish the peer. + async fn sanitize_v1_assignments( + &mut self, + peer_id: PeerId, + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, + ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { + let mut sanitized_assignments = Vec::new(); + for (cert, candidate_index) in assignments.into_iter() { + let cert_bitfield_bits = match cert.cert.kind { + AssignmentCertKind::RelayVRFDelay { core_index } => core_index.0 as usize + 1, + // We don't want to run the VRF yet, but the output is always bounded by `n_cores`. + // We assume `candidate_bitfield` length for the core bitfield and we just check + // against `MAX_BITFIELD_SIZE` later. + AssignmentCertKind::RelayVRFModulo { .. } => candidate_index as usize + 1, + }; + + let candidate_bitfield_bits = candidate_index as usize + 1; + + // Ensure bitfields length under hard limit. + if cert_bitfield_bits > MAX_BITFIELD_SIZE || candidate_bitfield_bits > MAX_BITFIELD_SIZE + { + // Punish the peer for the invalid message. + modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) + .await; + } else { + sanitized_assignments.push((cert.into(), candidate_index.into())) + } + } + + sanitized_assignments + } + + // Filter out oversized candidate and certificate core bitfields. + // For each invalid assignment we also punish the peer. + async fn sanitize_v2_assignments( + &mut self, + peer_id: PeerId, + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + assignments: Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + ) -> Vec<(IndirectAssignmentCertV2, CandidateBitfield)> { + let mut sanitized_assignments = Vec::new(); + for (cert, candidate_bitfield) in assignments.into_iter() { + let cert_bitfield_bits = match &cert.cert.kind { + AssignmentCertKindV2::RelayVRFDelay { core_index } => core_index.0 as usize + 1, + // We don't want to run the VRF yet, but the output is always bounded by `n_cores`. + // We assume `candidate_bitfield` length for the core bitfield and we just check + // against `MAX_BITFIELD_SIZE` later. + AssignmentCertKindV2::RelayVRFModulo { .. } => candidate_bitfield.len(), + AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield } => + core_bitfield.len(), + }; + + let candidate_bitfield_bits = candidate_bitfield.len(); + + // Our bitfield has `Lsb0`. + let msb = candidate_bitfield_bits - 1; + + // Ensure bitfields length under hard limit. + if cert_bitfield_bits > MAX_BITFIELD_SIZE + || candidate_bitfield_bits > MAX_BITFIELD_SIZE + // Ensure minimum bitfield size - MSB needs to be one. + || !candidate_bitfield.bit_at(msb.as_bit_index()) + { + // Punish the peer for the invalid message. + modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) + .await; + } else { + sanitized_assignments.push((cert, candidate_bitfield)) + } + } + + sanitized_assignments + } } // This adjusts the required routing of messages in blocks that pass the block filter @@ -1753,14 +2022,14 @@ impl State { #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] async fn adjust_required_routing_and_propagate( ctx: &mut Context, - peer_data: &HashMap, blocks: &mut HashMap, topologies: &SessionGridTopologies, block_filter: BlockFilter, routing_modifier: RoutingModifier, + peer_views: &HashMap, ) where BlockFilter: Fn(&mut BlockEntry) -> bool, - RoutingModifier: Fn(&mut RequiredRouting, bool, &ValidatorIndex), + RoutingModifier: Fn(&RequiredRouting, bool, &ValidatorIndex) -> RequiredRouting, { let mut peer_assignments = HashMap::new(); let mut peer_approvals = HashMap::new(); @@ -1772,64 +2041,55 @@ async fn adjust_required_routing_and_propagate t, + None => continue, + }; - if message_state.required_routing.is_empty() { - continue - } + // We just need to iterate the `approval_entries` of the block entry as these contain all + // assignments that also link all approval votes. + for approval_entry in block_entry.approval_entries.values_mut() { + let new_required_routing = routing_modifier( + &approval_entry.routing_info().required_routing, + approval_entry.routing_info().local, + &approval_entry.validator_index(), + ); - let topology = match topologies.get_topology(block_entry.session) { - Some(t) => t, - None => continue, - }; + approval_entry.update_required_routing(new_required_routing); - // Propagate the message to all peers in the required routing set. - let message_subject = MessageSubject(*block_hash, candidate_index, *validator); + if approval_entry.routing_info().required_routing.is_empty() { + continue + } - let assignment_message = ( - IndirectAssignmentCert { - block_hash: *block_hash, - validator: *validator, - cert: message_state.approval_state.assignment_cert().clone(), - }, - candidate_index, - ); - let approval_message = - message_state.approval_state.approval_signature().map(|signature| { - IndirectSignedApprovalVote { - block_hash: *block_hash, - validator: *validator, - candidate_index, - signature, - } - }); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(*block_hash); for (peer, peer_knowledge) in &mut block_entry.known_by { if !topology .local_grid_neighbors() - .route_to_peer(message_state.required_routing, peer) + .route_to_peer(approval_entry.routing_info().required_routing, peer) { continue } - if !peer_knowledge.contains(&message_subject, MessageKind::Assignment) { - peer_knowledge.sent.insert(message_subject.clone(), MessageKind::Assignment); + // Only send stuff a peer doesn't know in the context of a relay chain block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge.clone(), message_kind); peer_assignments .entry(*peer) .or_insert_with(Vec::new) .push(assignment_message.clone()); } - if let Some(approval_message) = approval_message.as_ref() { - if !peer_knowledge.contains(&message_subject, MessageKind::Approval) { - peer_knowledge.sent.insert(message_subject.clone(), MessageKind::Approval); + // Filter approval votes. + for approval_message in &approval_messages { + let (approval_knowledge, message_kind) = approval_entry + .create_approval_knowledge(*block_hash, approval_message.candidate_index); + + if !peer_knowledge.contains(&approval_knowledge, message_kind) { + peer_knowledge.sent.insert(approval_knowledge, message_kind); peer_approvals .entry(*peer) .or_insert_with(Vec::new) @@ -1841,24 +2101,32 @@ async fn adjust_required_routing_and_propagate continue, - Some(v) => v, - }; - - send_assignments_batched(ctx.sender(), assignments_packet, peer, peer_protocol_version) + if let Some(peer_view) = peer_views.get(&peer) { + send_assignments_batched( + ctx.sender(), + assignments_packet, + &vec![(peer, peer_view.version)], + ) .await; + } else { + // This should never happen. + gum::warn!(target: LOG_TARGET, ?peer, "Unknown protocol version for peer",); + } } for (peer, approvals_packet) in peer_approvals { - let peer_protocol_version = match peer_data.get(&peer).map(|pd| pd.version) { - None => continue, - Some(v) => v, - }; - - send_approvals_batched(ctx.sender(), approvals_packet, peer, peer_protocol_version).await; + if let Some(peer_view) = peer_views.get(&peer) { + send_approvals_batched( + ctx.sender(), + approvals_packet, + &vec![(peer, peer_view.version)], + ) + .await; + } else { + // This should never happen. + gum::warn!(target: LOG_TARGET, ?peer, "Unknown protocol version for peer",); + } } } @@ -1960,12 +2228,21 @@ impl ApprovalDistribution { ApprovalDistributionMessage::NewBlocks(metas) => { state.handle_new_blocks(ctx, metrics, metas, rng).await; }, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index) => { + ApprovalDistributionMessage::DistributeAssignment(cert, candidate_indices) => { + let _span = state + .spans + .get(&cert.block_hash) + .map(|span| span.child("import-and-distribute-assignment")) + .unwrap_or_else(|| jaeger::Span::new(&cert.block_hash, "distribute-assignment")) + .with_string_tag("block-hash", format!("{:?}", cert.block_hash)) + .with_stage(jaeger::Stage::ApprovalDistribution); + gum::debug!( target: LOG_TARGET, - "Distributing our assignment on candidate (block={}, index={})", - cert.block_hash, - candidate_index, + ?candidate_indices, + block_hash = ?cert.block_hash, + assignment_kind = ?cert.cert.kind, + "Distributing our assignment on candidates", ); state @@ -1974,7 +2251,7 @@ impl ApprovalDistribution { &metrics, MessageSource::Local, cert, - candidate_index, + candidate_indices, rng, ) .await; @@ -2008,49 +2285,6 @@ impl ApprovalDistribution { } } -fn versioned_approvals_packet( - version: ValidationVersion, - approvals: Vec, -) -> VersionedValidationProtocol { - match version { - ValidationVersion::V1 => - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals), - )), - ValidationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals), - )), - } -} - -fn versioned_assignments_packet( - version: ValidationVersion, - assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, -) -> VersionedValidationProtocol { - match version { - ValidationVersion::V1 => - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments), - )), - ValidationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments), - )), - } -} - -fn filter_peers_by_version( - peers: &[(PeerId, ValidationVersion)], - version: ValidationVersion, -) -> Vec { - peers - .iter() - .filter(|(_, v)| v == &version) - .map(|(peer_id, _)| *peer_id) - .collect() -} - #[overseer::subsystem(ApprovalDistribution, error=SubsystemError, prefix=self::overseer)] impl ApprovalDistribution { fn start(self, ctx: Context) -> SpawnedSubsystem { @@ -2075,7 +2309,7 @@ const fn ensure_size_not_zero(size: usize) -> usize { /// the protocol configuration. pub const MAX_ASSIGNMENT_BATCH_SIZE: usize = ensure_size_not_zero( MAX_NOTIFICATION_SIZE as usize / - std::mem::size_of::<(IndirectAssignmentCert, CandidateIndex)>() / + std::mem::size_of::<(IndirectAssignmentCertV2, CandidateIndex)>() / 3, ); @@ -2084,6 +2318,54 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( MAX_NOTIFICATION_SIZE as usize / std::mem::size_of::() / 3, ); +// Low level helper for sending assignments. +async fn send_assignments_batched_inner( + sender: &mut impl overseer::ApprovalDistributionSenderTrait, + batch: impl IntoIterator, + peers: &[PeerId], + peer_version: ValidationVersion, +) { + let peers = peers.into_iter().cloned().collect::>(); + if peer_version == ValidationVersion::VStaging { + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments( + batch.into_iter().collect(), + ), + )), + )) + .await; + } else { + // Create a batch of v1 assignments from v2 assignments that are compatible with v1. + // `IndirectAssignmentCertV2` -> `IndirectAssignmentCert` + let batch = batch + .into_iter() + .filter_map(|(cert, candidates)| { + cert.try_into().ok().map(|cert| { + ( + cert, + // First 1 bit index is the candidate index. + candidates + .first_one() + .map(|index| index as CandidateIndex) + .expect("Assignment was checked for not being empty; qed"), + ) + }) + }) + .collect(); + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Assignments(batch), + )), + )) + .await; + } +} + /// Send assignments while honoring the `max_notification_size` of the protocol. /// /// Splitting the messages into multiple notifications allows more granular processing at the @@ -2091,37 +2373,81 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( /// of assignments and can `select!` other tasks. pub(crate) async fn send_assignments_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - assignments: Vec<(IndirectAssignmentCert, CandidateIndex)>, - peer: PeerId, - protocol_version: ValidationVersion, + v2_assignments: impl IntoIterator + Clone, + peers: &[(PeerId, ProtocolVersion)], ) { - let mut batches = assignments.into_iter().peekable(); + let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + // Older peers(v1) do not understand `AssignmentsV2` messages, so we have to filter these + // out. + let v1_assignments = v2_assignments + .clone() + .into_iter() + .filter(|(_, candidates)| candidates.count_ones() == 1); + + let mut v1_batches = v1_assignments.peekable(); + + while v1_batches.peek().is_some() { + let batch: Vec<_> = v1_batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect(); + send_assignments_batched_inner(sender, batch, &v1_peers, ValidationVersion::V1).await; + } + } - while batches.peek().is_some() { - let batch: Vec<_> = batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect(); - let versioned = versioned_assignments_packet(protocol_version, batch); + if !v2_peers.is_empty() { + let mut v2_batches = v2_assignments.into_iter().peekable(); - sender - .send_message(NetworkBridgeTxMessage::SendValidationMessage(vec![peer], versioned)) - .await; + while v2_batches.peek().is_some() { + let batch = v2_batches.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect::>(); + send_assignments_batched_inner(sender, batch, &v2_peers, ValidationVersion::VStaging) + .await; + } } } -/// Send approvals while honoring the `max_notification_size` of the protocol. +/// Send approvals while honoring the `max_notification_size` of the protocol and peer version. pub(crate) async fn send_approvals_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - approvals: Vec, - peer: PeerId, - protocol_version: ValidationVersion, + approvals: impl IntoIterator + Clone, + peers: &[(PeerId, ProtocolVersion)], ) { - let mut batches = approvals.into_iter().peekable(); - - while batches.peek().is_some() { - let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); - let versioned = versioned_approvals_packet(protocol_version, batch); + let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); + let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + + if !v1_peers.is_empty() { + let mut batches = approvals.clone().into_iter().peekable(); + + while batches.peek().is_some() { + let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); + + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + v1_peers.clone(), + Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( + protocol_v1::ApprovalDistributionMessage::Approvals(batch), + )), + )) + .await; + } + } - sender - .send_message(NetworkBridgeTxMessage::SendValidationMessage(vec![peer], versioned)) - .await; + if !v2_peers.is_empty() { + let mut batches = approvals.into_iter().peekable(); + + while batches.peek().is_some() { + let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); + + sender + .send_message(NetworkBridgeTxMessage::SendValidationMessage( + v2_peers.clone(), + Versioned::VStaging( + protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(batch), + ), + ), + )) + .await; + } } } diff --git a/polkadot/node/network/approval-distribution/src/metrics.rs b/polkadot/node/network/approval-distribution/src/metrics.rs index 896866ce099a..6864259e6fdb 100644 --- a/polkadot/node/network/approval-distribution/src/metrics.rs +++ b/polkadot/node/network/approval-distribution/src/metrics.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . use polkadot_node_metrics::metrics::{prometheus, Metrics as MetricsTrait}; +use polkadot_node_primitives::approval::v2::AssignmentCertKindV2; /// Approval Distribution metrics. #[derive(Default, Clone)] @@ -22,21 +23,34 @@ pub struct Metrics(Option); #[derive(Clone)] struct MetricsInner { - assignments_imported_total: prometheus::Counter, + assignments_imported_total: prometheus::CounterVec, approvals_imported_total: prometheus::Counter, unified_with_peer_total: prometheus::Counter, aggression_l1_messages_total: prometheus::Counter, aggression_l2_messages_total: prometheus::Counter, - time_unify_with_peer: prometheus::Histogram, time_import_pending_now_known: prometheus::Histogram, time_awaiting_approval_voting: prometheus::Histogram, } +trait AsLabel { + fn as_label(&self) -> &str; +} + +impl AsLabel for &AssignmentCertKindV2 { + fn as_label(&self) -> &str { + match self { + AssignmentCertKindV2::RelayVRFDelay { .. } => "VRF Delay", + AssignmentCertKindV2::RelayVRFModulo { .. } => "VRF Modulo", + AssignmentCertKindV2::RelayVRFModuloCompact { .. } => "VRF Modulo Compact", + } + } +} + impl Metrics { - pub(crate) fn on_assignment_imported(&self) { + pub(crate) fn on_assignment_imported(&self, kind: &AssignmentCertKindV2) { if let Some(metrics) = &self.0 { - metrics.assignments_imported_total.inc(); + metrics.assignments_imported_total.with_label_values(&[kind.as_label()]).inc(); } } @@ -89,9 +103,12 @@ impl MetricsTrait for Metrics { fn try_register(registry: &prometheus::Registry) -> Result { let metrics = MetricsInner { assignments_imported_total: prometheus::register( - prometheus::Counter::new( - "polkadot_parachain_assignments_imported_total", - "Number of valid assignments imported locally or from other peers.", + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_assignments_imported_total", + "Number of valid assignments imported locally or from other peers.", + ), + &["kind"], )?, registry, )?, @@ -124,10 +141,16 @@ impl MetricsTrait for Metrics { registry, )?, time_unify_with_peer: prometheus::register( - prometheus::Histogram::with_opts(prometheus::HistogramOpts::new( - "polkadot_parachain_time_unify_with_peer", - "Time spent within fn `unify_with_peer`.", - ).buckets(vec![0.000625, 0.00125,0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,]))?, + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_time_unify_with_peer", + "Time spent within fn `unify_with_peer`.", + ) + .buckets(vec![ + 0.000625, 0.00125, 0.0025, 0.005, 0.0075, 0.01, 0.025, 0.05, 0.1, 0.25, + 0.5, 1.0, 2.5, 5.0, 10.0, + ]), + )?, registry, )?, time_import_pending_now_known: prometheus::register( diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 1e9ae7b62007..f0c3c4f8ba64 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -24,20 +24,26 @@ use polkadot_node_network_protocol::{ view, ObservedRole, }; use polkadot_node_primitives::approval::{ - AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, + v1::{ + AssignmentCert, AssignmentCertKind, IndirectAssignmentCert, VrfOutput, VrfProof, + VrfSignature, + }, + v2::{ + AssignmentCertKindV2, AssignmentCertV2, CoreBitfield, IndirectAssignmentCertV2, + RELAY_VRF_MODULO_CONTEXT, + }, }; use polkadot_node_subsystem::messages::{ network_bridge_event, AllMessages, ApprovalCheckError, ReportPeerMessage, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt as _}; -use polkadot_primitives::{AuthorityDiscoveryId, BlakeTwo256, HashT}; +use polkadot_primitives::{AuthorityDiscoveryId, BlakeTwo256, CoreIndex, HashT}; use polkadot_primitives_test_helpers::dummy_signature; use rand::SeedableRng; use sp_authority_discovery::AuthorityPair as AuthorityDiscoveryPair; use sp_core::crypto::Pair as PairT; use std::time::Duration; - type VirtualOverseer = test_helpers::TestSubsystemContextHandle; fn test_harness>( @@ -219,15 +225,15 @@ async fn setup_gossip_topology( async fn setup_peer_with_view( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, - validation_version: ValidationVersion, view: View, + version: ValidationVersion, ) { overseer_send( virtual_overseer, ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerConnected( *peer_id, ObservedRole::Full, - validation_version.into(), + version.into(), None, )), ) @@ -244,12 +250,28 @@ async fn setup_peer_with_view( async fn send_message_from_peer( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, - msg: net_protocol::ApprovalDistributionMessage, + msg: protocol_v1::ApprovalDistributionMessage, +) { + overseer_send( + virtual_overseer, + ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( + *peer_id, + Versioned::V1(msg), + )), + ) + .await; +} + +async fn send_message_from_peer_v2( + virtual_overseer: &mut VirtualOverseer, + peer_id: &PeerId, + msg: protocol_vstaging::ApprovalDistributionMessage, ) { overseer_send( virtual_overseer, ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( - *peer_id, msg, + *peer_id, + Versioned::VStaging(msg), )), ) .await; @@ -273,6 +295,28 @@ fn fake_assignment_cert(block_hash: Hash, validator: ValidatorIndex) -> Indirect } } +fn fake_assignment_cert_v2( + block_hash: Hash, + validator: ValidatorIndex, + core_bitfield: CoreBitfield, +) -> IndirectAssignmentCertV2 { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"WhenParachains?"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + IndirectAssignmentCertV2 { + block_hash, + validator, + cert: AssignmentCertV2 { + kind: AssignmentCertKindV2::RelayVRFModuloCompact { core_bitfield }, + vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) }, + }, + } +} + async fn expect_reputation_change( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, @@ -331,9 +375,9 @@ fn try_import_the_same_assignment() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, ValidationVersion::V1, view![]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -353,7 +397,7 @@ fn try_import_the_same_assignment() { let assignments = vec![(cert.clone(), 0u32)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, &peer_a, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_a, msg).await; expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; @@ -362,10 +406,11 @@ fn try_import_the_same_assignment() { overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( assignment, - 0u32, + claimed_indices, tx, )) => { - assert_eq!(assignment, cert); + assert_eq!(claimed_indices, 0u32.into()); + assert_eq!(assignment, cert.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -385,12 +430,104 @@ fn try_import_the_same_assignment() { } ); - // setup new peer - setup_peer_with_view(overseer, &peer_d, ValidationVersion::V1, view![]).await; + // setup new peer with V2 + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; // send the same assignment from peer_d let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, &peer_d, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_d, msg).await; + + expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }); +} + +/// Just like `try_import_the_same_assignment` but use `VRFModuloCompact` assignments for multiple +/// cores. +#[test] +fn try_import_the_same_assignment_v2() { + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + let peer_c = PeerId::random(); + let peer_d = PeerId::random(); + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 2, + candidates: vec![Default::default(); 1], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // send the assignment related to `hash` + let validator_index = ValidatorIndex(0); + let cores = vec![1, 2, 3, 4]; + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + let assignments = vec![(cert.clone(), cores.clone().try_into().unwrap())]; + + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v2(overseer, &peer_a, msg).await; + + expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; + + // send an `Accept` message from the Approval Voting subsystem + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + assignment, + claimed_indices, + tx, + )) => { + assert_eq!(claimed_indices, cores.try_into().unwrap()); + assert_eq!(assignment, cert.into()); + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 2); + assert_eq!(assignments.len(), 1); + } + ); + + // setup new peer + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; + + // send the same assignment from peer_d + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer_v2(overseer, &peer_d, msg).await; expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; @@ -413,7 +550,7 @@ fn delay_reputation_change() { let overseer = &mut virtual_overseer; // Setup peers - setup_peer_with_view(overseer, &peer, ValidationVersion::V1, view![]).await; + setup_peer_with_view(overseer, &peer, view![], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -433,17 +570,18 @@ fn delay_reputation_change() { let assignments = vec![(cert.clone(), 0u32)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, &peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer, msg).await; // send an `Accept` message from the Approval Voting subsystem assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( assignment, - 0u32, + claimed_candidates, tx, )) => { - assert_eq!(assignment, cert); + assert_eq!(assignment.cert, cert.cert.into()); + assert_eq!(claimed_candidates, vec![0u32].try_into().unwrap()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -474,7 +612,7 @@ fn spam_attack_results_in_negative_reputation_change() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![]).await; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; // new block `hash_b` with 20 candidates let candidates_count = 20; @@ -501,7 +639,7 @@ fn spam_attack_results_in_negative_reputation_change() { .collect(); let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; for i in 0..candidates_count { expect_reputation_change(overseer, peer, COST_UNEXPECTED_MESSAGE).await; @@ -513,8 +651,8 @@ fn spam_attack_results_in_negative_reputation_change() { claimed_candidate_index, tx, )) => { - assert_eq!(assignment, assignments[i].0); - assert_eq!(claimed_candidate_index, assignments[i].1); + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -533,7 +671,7 @@ fn spam_attack_results_in_negative_reputation_change() { .await; // send the assignments again - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; // each of them will incur `COST_UNEXPECTED_MESSAGE`, not only the first one for _ in 0..candidates_count { @@ -558,7 +696,7 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![]).await; + setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; // new block `hash` with 1 candidates let meta = BlockApprovalMeta { @@ -578,7 +716,10 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let cert = fake_assignment_cert(hash, validator_index); overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -610,12 +751,12 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { // the peer could send us it as well let assignments = vec![(cert, candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "we should not punish the peer"); // send the assignments again - send_message_from_peer(overseer, peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, peer, msg).await; // now we should expect_reputation_change(overseer, peer, COST_DUPLICATE_MESSAGE).await; @@ -633,10 +774,10 @@ fn import_approval_happy_path() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; - // setup peers - setup_peer_with_view(overseer, &peer_a, ValidationVersion::V1, view![]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, ValidationVersion::V1, view![hash]).await; + // setup peers with V1 and V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -656,10 +797,14 @@ fn import_approval_happy_path() { let cert = fake_assignment_cert(hash, validator_index); overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; + // 1 peer is v1 assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( @@ -668,7 +813,21 @@ fn import_approval_happy_path() { protocol_v1::ApprovalDistributionMessage::Assignments(assignments) )) )) => { - assert_eq!(peers.len(), 2); + assert_eq!(peers.len(), 1); + assert_eq!(assignments.len(), 1); + } + ); + + // 1 peer is v2 + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert_eq!(peers.len(), 1); assert_eq!(assignments.len(), 1); } ); @@ -681,7 +840,7 @@ fn import_approval_happy_path() { signature: dummy_signature(), }; let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -722,8 +881,8 @@ fn import_approval_bad() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, ValidationVersion::V1, view![]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -749,14 +908,14 @@ fn import_approval_bad() { signature: dummy_signature(), }; let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; // now import an assignment from peer_b let assignments = vec![(cert.clone(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -765,8 +924,8 @@ fn import_approval_bad() { i, tx, )) => { - assert_eq!(assignment, cert); - assert_eq!(i, candidate_index); + assert_eq!(assignment, cert.into()); + assert_eq!(i, candidate_index.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -775,7 +934,7 @@ fn import_approval_bad() { // and try again let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -911,12 +1070,20 @@ fn update_peer_view() { let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(0)); let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(0)); - overseer_send(overseer, ApprovalDistributionMessage::DistributeAssignment(cert_a, 0)).await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_a.into(), 0.into()), + ) + .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeAssignment(cert_b, 0)).await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment(cert_b.into(), 0.into()), + ) + .await; // connect a peer - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash_a]).await; + setup_peer_with_view(overseer, peer, view![hash_a], ValidationVersion::V1).await; // we should send relevant assignments to the peer assert_matches!( @@ -934,7 +1101,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!(state.peer_data.get(peer).map(|data| data.view.finalized_number), Some(0)); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(0)); assert_eq!( state .blocks @@ -965,7 +1132,7 @@ fn update_peer_view() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert_c.clone(), 0), + ApprovalDistributionMessage::DistributeAssignment(cert_c.clone().into(), 0.into()), ) .await; @@ -986,7 +1153,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!(state.peer_data.get(peer).map(|data| data.view.finalized_number), Some(2)); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(2)); assert_eq!( state .blocks @@ -1016,10 +1183,7 @@ fn update_peer_view() { virtual_overseer }); - assert_eq!( - state.peer_data.get(peer).map(|data| data.view.finalized_number), - Some(finalized_number) - ); + assert_eq!(state.peer_views.get(peer).map(|v| v.view.finalized_number), Some(finalized_number)); assert!(state.blocks.get(&hash_c).unwrap().known_by.get(peer).is_none()); } @@ -1034,7 +1198,7 @@ fn import_remotely_then_locally() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup the peer - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -1054,7 +1218,7 @@ fn import_remotely_then_locally() { let cert = fake_assignment_cert(hash, validator_index); let assignments = vec![(cert.clone(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, peer, msg).await; // send an `Accept` message from the Approval Voting subsystem assert_matches!( @@ -1064,8 +1228,8 @@ fn import_remotely_then_locally() { i, tx, )) => { - assert_eq!(assignment, cert); - assert_eq!(i, candidate_index); + assert_eq!(assignment, cert.clone().into()); + assert_eq!(i, candidate_index.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -1075,7 +1239,10 @@ fn import_remotely_then_locally() { // import the same assignment locally overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1089,7 +1256,7 @@ fn import_remotely_then_locally() { signature: dummy_signature(), }; let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, peer, Versioned::V1(msg)).await; + send_message_from_peer(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1147,7 +1314,10 @@ fn sends_assignments_even_when_state_is_approved() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1155,7 +1325,7 @@ fn sends_assignments_even_when_state_is_approved() { .await; // connect the peer. - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1191,6 +1361,112 @@ fn sends_assignments_even_when_state_is_approved() { }); } +/// Same as `sends_assignments_even_when_state_is_approved_v2` but with `VRFModuloCompact` +/// assignemnts. +#[test] +fn sends_assignments_even_when_state_is_approved_v2() { + let peer_a = PeerId::random(); + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + let peer = &peer_a; + + let _ = test_harness(State::default(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 4], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + let validator_index = ValidatorIndex(0); + let cores = vec![0, 1, 2, 3]; + let candidate_bitfield: CandidateBitfield = cores.clone().try_into().unwrap(); + + let core_bitfield: CoreBitfield = cores + .iter() + .map(|index| CoreIndex(*index)) + .collect::>() + .try_into() + .unwrap(); + + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); + + // Assumes candidate index == core index. + let approvals = cores + .iter() + .map(|core| IndirectSignedApprovalVote { + block_hash: hash, + candidate_index: *core, + validator: validator_index, + signature: dummy_signature(), + }) + .collect::>(); + + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_bitfield.clone(), + ), + ) + .await; + + for approval in &approvals { + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone()), + ) + .await; + } + + // connect the peer. + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::VStaging).await; + + let assignments = vec![(cert.clone(), candidate_bitfield.clone())]; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(sent_assignments) + )) + )) => { + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_assignments, assignments); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(sent_approvals) + )) + )) => { + // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a + // hashmap as well. + let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); + let approvals = approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); + + assert_eq!(peers, vec![*peer]); + assert_eq!(sent_approvals, approvals); + } + ); + + assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); + virtual_overseer + }); +} + /// /// /// 1. Receive remote peer view update with an unknown head @@ -1219,7 +1495,7 @@ fn race_condition_in_local_vs_remote_view_update() { }; // This will send a peer view that is ahead of our view - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash_b]).await; + setup_peer_with_view(overseer, peer, view![hash_b], ValidationVersion::V1).await; // Send our view update to include a new head overseer_send( @@ -1240,7 +1516,7 @@ fn race_condition_in_local_vs_remote_view_update() { .collect(); let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer(overseer, peer, Versioned::V1(msg.clone())).await; + send_message_from_peer(overseer, peer, msg.clone()).await; // This will handle pending messages being processed let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); @@ -1257,8 +1533,8 @@ fn race_condition_in_local_vs_remote_view_update() { claimed_candidate_index, tx, )) => { - assert_eq!(assignment, assignments[i].0); - assert_eq!(claimed_candidate_index, assignments[i].1); + assert_eq!(assignment, assignments[i].0.clone().into()); + assert_eq!(claimed_candidate_index, assignments[i].1.into()); tx.send(AssignmentCheckResult::Accepted).unwrap(); } ); @@ -1283,7 +1559,7 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. @@ -1325,7 +1601,10 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1388,7 +1667,7 @@ fn propagates_assignments_along_unshared_dimension() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. @@ -1424,7 +1703,7 @@ fn propagates_assignments_along_unshared_dimension() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -1473,7 +1752,7 @@ fn propagates_assignments_along_unshared_dimension() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -1530,7 +1809,7 @@ fn propagates_to_required_after_connect() { // Connect all peers except omitted. for (i, (peer, _)) in peers.iter().enumerate() { if !omitted.contains(&i) { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } } @@ -1573,7 +1852,10 @@ fn propagates_to_required_after_connect() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1619,7 +1901,7 @@ fn propagates_to_required_after_connect() { ); for i in omitted.iter().copied() { - setup_peer_with_view(overseer, &peers[i].0, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, &peers[i].0, view![hash], ValidationVersion::V1).await; assert_matches!( overseer_recv(overseer).await, @@ -1668,7 +1950,7 @@ fn sends_to_more_peers_after_getting_topology() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -1698,7 +1980,10 @@ fn sends_to_more_peers_after_getting_topology() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1820,7 +2105,7 @@ fn originator_aggression_l1() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -1857,7 +2142,10 @@ fn originator_aggression_l1() { overseer_send( overseer, - ApprovalDistributionMessage::DistributeAssignment(cert.clone(), candidate_index), + ApprovalDistributionMessage::DistributeAssignment( + cert.clone().into(), + candidate_index.into(), + ), ) .await; @@ -1979,7 +2267,7 @@ fn non_originator_aggression_l1() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -2008,12 +2296,12 @@ fn non_originator_aggression_l1() { ) .await; - let assignments = vec![(cert.clone(), candidate_index)]; + let assignments = vec![(cert.clone().into(), candidate_index)]; let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments.clone()); // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -2084,7 +2372,7 @@ fn non_originator_aggression_l2() { // Connect all peers except omitted. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // new block `hash_a` with 1 candidates @@ -2118,7 +2406,7 @@ fn non_originator_aggression_l2() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -2249,7 +2537,7 @@ fn resends_messages_periodically() { // Connect all peers. for (peer, _) in &peers { - setup_peer_with_view(overseer, peer, ValidationVersion::V1, view![hash]).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; } // Set up a gossip topology. @@ -2284,7 +2572,7 @@ fn resends_messages_periodically() { // Issuer of the message is important, not the peer we receive from. // 99 deliberately chosen because it's not in X or Y. - send_message_from_peer(overseer, &peers[99].0, Versioned::V1(msg)).await; + send_message_from_peer(overseer, &peers[99].0, msg).await; assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( @@ -2375,126 +2663,6 @@ fn resends_messages_periodically() { }); } -/// Tests that peers correctly receive versioned messages. -#[test] -fn import_versioned_approval() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); - let parent_hash = Hash::repeat_byte(0xFF); - let hash = Hash::repeat_byte(0xAA); - - let state = state_without_reputation_delay(); - let _ = test_harness(state, |mut virtual_overseer| async move { - let overseer = &mut virtual_overseer; - // All peers are aware of relay parent. - setup_peer_with_view(overseer, &peer_a, ValidationVersion::VStaging, view![hash]).await; - setup_peer_with_view(overseer, &peer_b, ValidationVersion::V1, view![hash]).await; - setup_peer_with_view(overseer, &peer_c, ValidationVersion::VStaging, view![hash]).await; - - // new block `hash_a` with 1 candidates - let meta = BlockApprovalMeta { - hash, - parent_hash, - number: 1, - candidates: vec![Default::default(); 1], - slot: 1.into(), - session: 1, - }; - let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); - overseer_send(overseer, msg).await; - - // import an assignment related to `hash` locally - let validator_index = ValidatorIndex(0); - let candidate_index = 0u32; - let cert = fake_assignment_cert(hash, validator_index); - overseer_send( - overseer, - ApprovalDistributionMessage::DistributeAssignment(cert, candidate_index), - ) - .await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers, vec![peer_b]); - assert_eq!(assignments.len(), 1); - } - ); - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) - )) - )) => { - assert_eq!(peers.len(), 2); - assert!(peers.contains(&peer_a)); - assert!(peers.contains(&peer_c)); - - assert_eq!(assignments.len(), 1); - } - ); - - // send the an approval from peer_a - let approval = IndirectSignedApprovalVote { - block_hash: hash, - candidate_index, - validator: validator_index, - signature: dummy_signature(), - }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_a, Versioned::VStaging(msg)).await; - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( - vote, - tx, - )) => { - assert_eq!(vote, approval); - tx.send(ApprovalCheckResult::Accepted).unwrap(); - } - ); - - expect_reputation_change(overseer, &peer_a, BENEFIT_VALID_MESSAGE_FIRST).await; - - // Peers b and c receive versioned approval messages. - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers, vec![peer_b]); - assert_eq!(approvals.len(), 1); - } - ); - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) - )) - )) => { - assert_eq!(peers, vec![peer_c]); - assert_eq!(approvals.len(), 1); - } - ); - virtual_overseer - }); -} - fn batch_test_round(message_count: usize) { use polkadot_node_subsystem::SubsystemContext; let pool = sp_core::testing::TaskExecutor::new(); @@ -2512,7 +2680,9 @@ fn batch_test_round(message_count: usize) { let validators = 0..message_count; let assignments: Vec<_> = validators .clone() - .map(|index| (fake_assignment_cert(Hash::zero(), ValidatorIndex(index as u32)), 0)) + .map(|index| { + (fake_assignment_cert(Hash::zero(), ValidatorIndex(index as u32)).into(), 0.into()) + }) .collect(); let approvals: Vec<_> = validators @@ -2525,9 +2695,18 @@ fn batch_test_round(message_count: usize) { .collect(); let peer = PeerId::random(); - send_assignments_batched(&mut sender, assignments.clone(), peer, ValidationVersion::V1) - .await; - send_approvals_batched(&mut sender, approvals.clone(), peer, ValidationVersion::V1).await; + send_assignments_batched( + &mut sender, + assignments.clone(), + &vec![(peer, ValidationVersion::V1.into())], + ) + .await; + send_approvals_batched( + &mut sender, + approvals.clone(), + &vec![(peer, ValidationVersion::V1.into())], + ) + .await; // Check expected assignments batches. for assignment_index in (0..assignments.len()).step_by(super::MAX_ASSIGNMENT_BATCH_SIZE) { @@ -2549,7 +2728,7 @@ fn batch_test_round(message_count: usize) { assert_eq!(peers.len(), 1); for (message_index, assignment) in sent_assignments.iter().enumerate() { - assert_eq!(assignment.0, assignments[assignment_index + message_index].0); + assert_eq!(assignment.0, assignments[assignment_index + message_index].0.clone().try_into().unwrap()); assert_eq!(assignment.1, 0); } } diff --git a/polkadot/node/network/bitfield-distribution/src/lib.rs b/polkadot/node/network/bitfield-distribution/src/lib.rs index c85d874bc4db..55243edd3a5e 100644 --- a/polkadot/node/network/bitfield-distribution/src/lib.rs +++ b/polkadot/node/network/bitfield-distribution/src/lib.rs @@ -131,9 +131,9 @@ pub struct PeerData { /// Data used to track information of peers and relay parents the /// overseer ordered us to work on. -#[derive(Default, Debug)] +#[derive(Default)] struct ProtocolState { - /// Track all active peers and their views + /// Track all active peer views and protocol versions /// to determine what is relevant to them. peer_data: HashMap, @@ -775,9 +775,11 @@ async fn handle_network_msg( handle_peer_view_change(ctx, state, new_peer, old_view, rng).await; } }, - NetworkBridgeEvent::PeerViewChange(peerid, new_view) => { - gum::trace!(target: LOG_TARGET, ?peerid, ?new_view, "Peer view change"); - handle_peer_view_change(ctx, state, peerid, new_view, rng).await; + NetworkBridgeEvent::PeerViewChange(peer_id, new_view) => { + gum::trace!(target: LOG_TARGET, ?peer_id, ?new_view, "Peer view change"); + if state.peer_data.get(&peer_id).is_some() { + handle_peer_view_change(ctx, state, peer_id, new_view, rng).await; + } }, NetworkBridgeEvent::OurViewChange(new_view) => { gum::trace!(target: LOG_TARGET, ?new_view, "Our view change"); diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index 4f21212dcb64..4d2d1fba4891 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -28,23 +28,109 @@ use sc_network::{ }; use polkadot_node_network_protocol::{ - peer_set::{PeerSet, PeerSetProtocolNames, ProtocolVersion}, + peer_set::{ + CollationVersion, PeerSet, PeerSetProtocolNames, ProtocolVersion, ValidationVersion, + }, request_response::{OutgoingRequest, Recipient, ReqProtocolNames, Requests}, - PeerId, + v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, }; use polkadot_primitives::{AuthorityDiscoveryId, Block, Hash}; -use crate::validator_discovery::AuthorityDiscovery; +use crate::{metrics::Metrics, validator_discovery::AuthorityDiscovery, WireMessage}; // network bridge network abstraction log target const LOG_TARGET: &'static str = "parachain::network-bridge-net"; -/// Send a message to the network. +// Helper function to send a validation v1 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_validation_message_v1( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v1 message to peers",); + + send_message( + net, + peers, + PeerSet::Validation, + ValidationVersion::V1.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +// Helper function to send a validation v2 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_validation_message_v2( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v2 message to peers",); + + send_message( + net, + peers, + PeerSet::Validation, + ValidationVersion::VStaging.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +// Helper function to send a collation v1 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_collation_message_v1( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + send_message( + net, + peers, + PeerSet::Collation, + CollationVersion::V1.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +// Helper function to send a collation v2 message to a list of peers. +// Messages are always sent via the main protocol, even legacy protocol messages. +pub(crate) fn send_collation_message_v2( + net: &mut impl Network, + peers: Vec, + peerset_protocol_names: &PeerSetProtocolNames, + message: WireMessage, + metrics: &Metrics, +) { + send_message( + net, + peers, + PeerSet::Collation, + CollationVersion::VStaging.into(), + peerset_protocol_names, + message, + metrics, + ); +} + +/// Lower level function that sends a message to the network using the main protocol version. /// /// This function is only used internally by the network-bridge, which is responsible to only send /// messages that are compatible with the passed peer set, as that is currently not enforced by /// this function. These are messages of type `WireMessage` parameterized on the matching type. -pub(crate) fn send_message( +fn send_message( net: &mut impl Network, mut peers: Vec, peer_set: PeerSet, @@ -64,6 +150,17 @@ pub(crate) fn send_message( encoded }; + // optimization: generate the protocol name once. + let protocol_name = protocol_names.get_name(peer_set, version); + gum::trace!( + target: LOG_TARGET, + ?peers, + ?version, + ?protocol_name, + ?message, + "Sending message to peers", + ); + // optimization: avoid cloning the message for the last peer in the // list. The message payload can be quite large. If the underlying // network used `Bytes` this would not be necessary. diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 002919c5b0e5..15ed6b48745d 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -64,9 +64,11 @@ use super::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// /// Defines the `Network` trait with an implementation for an `Arc`. -use crate::network::{send_message, Network}; - -use crate::network::get_peer_id_by_authority_id; +use crate::network::{ + send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, + send_validation_message_v2, Network, +}; +use crate::{network::get_peer_id_by_authority_id, WireMessage}; use super::metrics::Metrics; @@ -250,22 +252,18 @@ where match ValidationVersion::try_from(version) .expect("try_get_protocol has already checked version is known; qed") { - ValidationVersion::V1 => send_message( + ValidationVersion::V1 => send_validation_message_v1( &mut network_service, vec![peer], - PeerSet::Validation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, ), &metrics, ), - ValidationVersion::VStaging => send_message( + ValidationVersion::VStaging => send_validation_message_v2( &mut network_service, vec![peer], - PeerSet::Validation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, @@ -292,22 +290,18 @@ where match CollationVersion::try_from(version) .expect("try_get_protocol has already checked version is known; qed") { - CollationVersion::V1 => send_message( + CollationVersion::V1 => send_collation_message_v1( &mut network_service, vec![peer], - PeerSet::Collation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, ), &metrics, ), - CollationVersion::VStaging => send_message( + CollationVersion::VStaging => send_collation_message_v2( &mut network_service, vec![peer], - PeerSet::Collation, - version, &peerset_protocol_names, WireMessage::::ViewUpdate( local_view, @@ -384,8 +378,16 @@ where .filter_map(|(protocol, msg_bytes)| { // version doesn't matter because we always receive on the 'correct' // protocol name, not the negotiated fallback. - let (peer_set, _version) = + let (peer_set, version) = peerset_protocol_names.try_get_protocol(protocol)?; + gum::trace!( + target: LOG_TARGET, + ?peer_set, + ?protocol, + ?version, + "Received notification" + ); + if peer_set == PeerSet::Validation { if expected_versions[PeerSet::Validation].is_none() { return Some(Err(UNCONNECTED_PEERSET_COST)) @@ -832,7 +834,7 @@ fn update_our_view( metrics, ); - send_validation_message_vstaging( + send_validation_message_v2( net, vstaging_validation_peers, peerset_protocol_names, @@ -840,7 +842,7 @@ fn update_our_view( metrics, ); - send_collation_message_vstaging( + send_collation_message_v2( net, vstaging_collation_peers, peerset_protocol_names, @@ -917,78 +919,6 @@ fn handle_peer_messages>( (outgoing_events, reports) } -fn send_validation_message_v1( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::V1.into(), - peerset_protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_v1( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::V1.into(), - peerset_protocol_names, - message, - metrics, - ); -} - -fn send_validation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} - async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeRxSenderTrait, diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index e0ca633547f4..7bd7f1e25c06 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -18,9 +18,7 @@ use super::*; use polkadot_node_network_protocol::{ - peer_set::{CollationVersion, PeerSet, PeerSetProtocolNames, ValidationVersion}, - request_response::ReqProtocolNames, - v1 as protocol_v1, vstaging as protocol_vstaging, PeerId, Versioned, + peer_set::PeerSetProtocolNames, request_response::ReqProtocolNames, Versioned, }; use polkadot_node_subsystem::{ @@ -40,7 +38,10 @@ use crate::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// /// Defines the `Network` trait with an implementation for an `Arc`. -use crate::network::{send_message, Network}; +use crate::network::{ + send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, + send_validation_message_v2, Network, +}; use crate::metrics::Metrics; @@ -186,6 +187,7 @@ where gum::trace!( target: LOG_TARGET, action = "SendValidationMessages", + ?msg, num_messages = 1usize, ); @@ -197,7 +199,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::VStaging(msg) => send_validation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -211,6 +213,7 @@ where target: LOG_TARGET, action = "SendValidationMessages", num_messages = %msgs.len(), + ?msgs, ); for (peers, msg) in msgs { @@ -222,7 +225,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::VStaging(msg) => send_validation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -247,7 +250,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -272,7 +275,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -373,75 +376,3 @@ where Ok(()) } - -fn send_validation_message_v1( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::V1.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_v1( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::V1.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_validation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Validation, - ValidationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} - -fn send_collation_message_vstaging( - net: &mut impl Network, - peers: Vec, - protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::VStaging.into(), - protocol_names, - message, - metrics, - ); -} diff --git a/polkadot/node/network/bridge/src/tx/tests.rs b/polkadot/node/network/bridge/src/tx/tests.rs index 21cd134c54f2..6e28a4032183 100644 --- a/polkadot/node/network/bridge/src/tx/tests.rs +++ b/polkadot/node/network/bridge/src/tx/tests.rs @@ -25,9 +25,9 @@ use std::collections::HashSet; use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSetProtocolNames, ValidationVersion}, request_response::{outgoing::Requests, ReqProtocolNames}, - ObservedRole, Versioned, + v1 as protocol_v1, vstaging as protocol_vstaging, ObservedRole, Versioned, }; use polkadot_node_subsystem::{FromOrchestra, OverseerSignal}; use polkadot_node_subsystem_test_helpers::TestSubsystemContextHandle; @@ -356,7 +356,6 @@ fn network_protocol_versioning_send() { } // send a validation protocol message. - { let approval_distribution_message = protocol_vstaging::ApprovalDistributionMessage::Approvals(Vec::new()); diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index 1bed2c12fe20..ba9b9a7f4900 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -313,7 +313,6 @@ macro_rules! impl_versioned_full_protocol_from { } }; } - /// Implement `TryFrom` for one versioned enum variant into the inner type. /// `$m_ty::$variant(inner) -> Ok(inner)` macro_rules! impl_versioned_try_from { @@ -441,7 +440,7 @@ pub mod v1 { }; use polkadot_node_primitives::{ - approval::{IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::v1::{IndirectAssignmentCert, IndirectSignedApprovalVote}, UncheckedSignedFullStatement, }; @@ -595,12 +594,15 @@ pub mod vstaging { use parity_scale_codec::{Decode, Encode}; use polkadot_primitives::vstaging::{ - CandidateHash, CandidateIndex, CollatorId, CollatorSignature, GroupIndex, Hash, - Id as ParaId, UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, + CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, + UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, }; use polkadot_node_primitives::{ - approval::{IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::{ + v1::IndirectSignedApprovalVote, + v2::{CandidateBitfield, IndirectAssignmentCertV2}, + }, UncheckedSignedFullStatement, }; @@ -769,10 +771,13 @@ pub mod vstaging { #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] pub enum ApprovalDistributionMessage { /// Assignments for candidates in recent, unfinalized blocks. + /// We use a bitfield to reference claimed candidates, where the bit index is equal to + /// candidate index. /// /// Actually checking the assignment may yield a different result. + /// TODO: Look at getting rid of bitfield in the future. #[codec(index = 0)] - Assignments(Vec<(IndirectAssignmentCert, CandidateIndex)>), + Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), /// Approvals for candidates in some recent, unfinalized block. #[codec(index = 1)] Approvals(Vec), @@ -841,3 +846,11 @@ pub mod vstaging { payload } } + +/// Returns the subset of `peers` with the specified `version`. +pub fn filter_by_peer_version( + peers: &[(PeerId, peer_set::ProtocolVersion)], + version: peer_set::ProtocolVersion, +) -> Vec { + peers.iter().filter(|(_, v)| v == &version).map(|(p, _)| *p).collect::>() +} diff --git a/polkadot/node/primitives/Cargo.toml b/polkadot/node/primitives/Cargo.toml index c7c323985510..170626b9cf41 100644 --- a/polkadot/node/primitives/Cargo.toml +++ b/polkadot/node/primitives/Cargo.toml @@ -21,6 +21,7 @@ polkadot-parachain = { path = "../../parachain", default-features = false } schnorrkel = "0.9.1" thiserror = "1.0.31" serde = { version = "1.0.163", features = ["derive"] } +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } [target.'cfg(not(target_os = "unknown"))'.dependencies] zstd = { version = "0.11.2", default-features = false } diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval.rs index 01f45a207874..bfa1f1aa47d2 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval.rs @@ -16,190 +16,515 @@ //! Types relevant for approval. -pub use sp_consensus_babe::{Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript}; - -use parity_scale_codec::{Decode, Encode}; -use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateIndex, CoreIndex, Hash, Header, SessionIndex, - ValidatorIndex, ValidatorSignature, -}; -use sp_application_crypto::ByteArray; -use sp_consensus_babe as babe_primitives; - -/// Validators assigning to check a particular candidate are split up into tranches. -/// Earlier tranches of validators check first, with later tranches serving as backup. -pub type DelayTranche = u32; - -/// A static context used to compute the Relay VRF story based on the -/// VRF output included in the header-chain. -pub const RELAY_VRF_STORY_CONTEXT: &[u8] = b"A&V RC-VRF"; - -/// A static context used for all relay-vrf-modulo VRFs. -pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD"; - -/// A static context used for all relay-vrf-modulo VRFs. -pub const RELAY_VRF_DELAY_CONTEXT: &[u8] = b"A&V DELAY"; - -/// A static context used for transcripts indicating assigned availability core. -pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED"; - -/// A static context associated with producing randomness for a core. -pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE"; - -/// A static context associated with producing randomness for a tranche. -pub const TRANCHE_RANDOMNESS_CONTEXT: &[u8] = b"A&V TRANCHE"; - -/// random bytes derived from the VRF submitted within the block by the -/// block author as a credential and used as input to approval assignment criteria. -#[derive(Debug, Clone, Encode, Decode, PartialEq)] -pub struct RelayVRFStory(pub [u8; 32]); - -/// Different kinds of input data or criteria that can prove a validator's assignment -/// to check a particular parachain. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub enum AssignmentCertKind { - /// An assignment story based on the VRF that authorized the relay-chain block where the - /// candidate was included combined with a sample number. +/// A list of primitives introduced in v1. +pub mod v1 { + use sp_consensus_babe as babe_primitives; + pub use sp_consensus_babe::{ + Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript, + }; + + use parity_scale_codec::{Decode, Encode}; + use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateIndex, CoreIndex, Hash, Header, SessionIndex, + ValidatorIndex, ValidatorSignature, + }; + use sp_application_crypto::ByteArray; + + /// Validators assigning to check a particular candidate are split up into tranches. + /// Earlier tranches of validators check first, with later tranches serving as backup. + pub type DelayTranche = u32; + + /// A static context used to compute the Relay VRF story based on the + /// VRF output included in the header-chain. + pub const RELAY_VRF_STORY_CONTEXT: &[u8] = b"A&V RC-VRF"; + + /// A static context used for all relay-vrf-modulo VRFs. + pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD"; + + /// A static context used for all relay-vrf-modulo VRFs. + pub const RELAY_VRF_DELAY_CONTEXT: &[u8] = b"A&V DELAY"; + + /// A static context used for transcripts indicating assigned availability core. + pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED"; + + /// A static context associated with producing randomness for a core. + pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE"; + + /// A static context associated with producing randomness for a tranche. + pub const TRANCHE_RANDOMNESS_CONTEXT: &[u8] = b"A&V TRANCHE"; + + /// random bytes derived from the VRF submitted within the block by the + /// block author as a credential and used as input to approval assignment criteria. + #[derive(Debug, Clone, Encode, Decode, PartialEq)] + pub struct RelayVRFStory(pub [u8; 32]); + + /// Different kinds of input data or criteria that can prove a validator's assignment + /// to check a particular parachain. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum AssignmentCertKind { + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`RELAY_VRF_DELAY_CONTEXT`] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + } + + /// A certification of assignment. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct AssignmentCert { + /// The criterion which is claimed to be met by this cert. + pub kind: AssignmentCertKind, + /// The VRF signature showing the criterion is met. + pub vrf: VrfSignature, + } + + /// An assignment criterion which refers to the candidate under which the assignment is + /// relevant by block hash. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectAssignmentCert { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The validator index. + pub validator: ValidatorIndex, + /// The cert itself. + pub cert: AssignmentCert, + } + + /// A signed approval vote which references the candidate indirectly via the block. /// - /// The context used to produce bytes is [`RELAY_VRF_MODULO_CONTEXT`] - RelayVRFModulo { - /// The sample number used in this cert. - sample: u32, - }, - /// An assignment story based on the VRF that authorized the relay-chain block where the - /// candidate was included combined with the index of a particular core. + /// In practice, we have a look-up from block hash and candidate index to candidate hash, + /// so this can be transformed into a `SignedApprovalVote`. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectSignedApprovalVote { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The index of the candidate in the list of candidates fully included as-of the block. + pub candidate_index: CandidateIndex, + /// The validator index. + pub validator: ValidatorIndex, + /// The signature by the validator. + pub signature: ValidatorSignature, + } + + /// Metadata about a block which is now live in the approval protocol. + #[derive(Debug)] + pub struct BlockApprovalMeta { + /// The hash of the block. + pub hash: Hash, + /// The number of the block. + pub number: BlockNumber, + /// The hash of the parent block. + pub parent_hash: Hash, + /// The candidates included by the block. + /// Note that these are not the same as the candidates that appear within the block body. + pub candidates: Vec, + /// The consensus slot of the block. + pub slot: Slot, + /// The session of the block. + pub session: SessionIndex, + } + + /// Errors that can occur during the approvals protocol. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum ApprovalError { + #[error("Schnorrkel signature error")] + SchnorrkelSignature(schnorrkel::errors::SignatureError), + #[error("Authority index {0} out of bounds")] + AuthorityOutOfBounds(usize), + } + + /// An unsafe VRF output. Provide BABE Epoch info to create a `RelayVRFStory`. + pub struct UnsafeVRFOutput { + vrf_output: VrfOutput, + slot: Slot, + authority_index: u32, + } + + impl UnsafeVRFOutput { + /// Get the slot. + pub fn slot(&self) -> Slot { + self.slot + } + + /// Compute the randomness associated with this VRF output. + pub fn compute_randomness( + self, + authorities: &[(babe_primitives::AuthorityId, babe_primitives::BabeAuthorityWeight)], + randomness: &babe_primitives::Randomness, + epoch_index: u64, + ) -> Result { + let author = match authorities.get(self.authority_index as usize) { + None => return Err(ApprovalError::AuthorityOutOfBounds(self.authority_index as _)), + Some(x) => &x.0, + }; + + let pubkey = schnorrkel::PublicKey::from_bytes(author.as_slice()) + .map_err(ApprovalError::SchnorrkelSignature)?; + + let transcript = + sp_consensus_babe::make_vrf_transcript(randomness, self.slot, epoch_index); + + let inout = self + .vrf_output + .0 + .attach_input_hash(&pubkey, transcript.0) + .map_err(ApprovalError::SchnorrkelSignature)?; + Ok(RelayVRFStory(inout.make_bytes(super::v1::RELAY_VRF_STORY_CONTEXT))) + } + } + + /// Extract the slot number and relay VRF from a header. /// - /// The context is [`RELAY_VRF_DELAY_CONTEXT`] - RelayVRFDelay { - /// The core index chosen in this cert. - core_index: CoreIndex, - }, -} + /// This fails if either there is no BABE `PreRuntime` digest or + /// the digest has type `SecondaryPlain`, which Substrate nodes do + /// not produce or accept anymore. + pub fn babe_unsafe_vrf_info(header: &Header) -> Option { + use babe_primitives::digests::CompatibleDigestItem; -/// A certification of assignment. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct AssignmentCert { - /// The criterion which is claimed to be met by this cert. - pub kind: AssignmentCertKind, - /// The VRF signature showing the criterion is met. - pub vrf: VrfSignature, -} + for digest in &header.digest.logs { + if let Some(pre) = digest.as_babe_pre_digest() { + let slot = pre.slot(); + let authority_index = pre.authority_index(); -/// An assignment criterion which refers to the candidate under which the assignment is -/// relevant by block hash. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct IndirectAssignmentCert { - /// A block hash where the candidate appears. - pub block_hash: Hash, - /// The validator index. - pub validator: ValidatorIndex, - /// The cert itself. - pub cert: AssignmentCert, -} + return pre.vrf_signature().map(|sig| UnsafeVRFOutput { + vrf_output: sig.output.clone(), + slot, + authority_index, + }) + } + } -/// A signed approval vote which references the candidate indirectly via the block. -/// -/// In practice, we have a look-up from block hash and candidate index to candidate hash, -/// so this can be transformed into a `SignedApprovalVote`. -#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -pub struct IndirectSignedApprovalVote { - /// A block hash where the candidate appears. - pub block_hash: Hash, - /// The index of the candidate in the list of candidates fully included as-of the block. - pub candidate_index: CandidateIndex, - /// The validator index. - pub validator: ValidatorIndex, - /// The signature by the validator. - pub signature: ValidatorSignature, + None + } } -/// Metadata about a block which is now live in the approval protocol. -#[derive(Debug)] -pub struct BlockApprovalMeta { - /// The hash of the block. - pub hash: Hash, - /// The number of the block. - pub number: BlockNumber, - /// The hash of the parent block. - pub parent_hash: Hash, - /// The candidates included by the block. - /// Note that these are not the same as the candidates that appear within the block body. - pub candidates: Vec, - /// The consensus slot of the block. - pub slot: Slot, - /// The session of the block. - pub session: SessionIndex, -} +/// A list of primitives introduced by v2. +pub mod v2 { + use parity_scale_codec::{Decode, Encode}; + pub use sp_consensus_babe::{ + Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript, + }; + use std::ops::BitOr; -/// Errors that can occur during the approvals protocol. -#[derive(Debug, thiserror::Error)] -#[allow(missing_docs)] -pub enum ApprovalError { - #[error("Schnorrkel signature error")] - SchnorrkelSignature(schnorrkel::errors::SignatureError), - #[error("Authority index {0} out of bounds")] - AuthorityOutOfBounds(usize), -} + use bitvec::{prelude::Lsb0, vec::BitVec}; + use polkadot_primitives::{CandidateIndex, CoreIndex, Hash, ValidatorIndex}; -/// An unsafe VRF output. Provide BABE Epoch info to create a `RelayVRFStory`. -pub struct UnsafeVRFOutput { - vrf_output: VrfOutput, - slot: Slot, - authority_index: u32, -} + /// A static context associated with producing randomness for a core. + pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE v2"; + /// A static context associated with producing randomness for v2 multi-core assignments. + pub const ASSIGNED_CORE_CONTEXT: &[u8] = b"A&V ASSIGNED v2"; + /// A static context used for all relay-vrf-modulo VRFs for v2 multi-core assignments. + pub const RELAY_VRF_MODULO_CONTEXT: &[u8] = b"A&V MOD v2"; + /// A read-only bitvec wrapper + #[derive(Clone, Debug, Encode, Decode, Hash, PartialEq, Eq)] + pub struct Bitfield(BitVec, std::marker::PhantomData); + + /// A `read-only`, `non-zero` bitfield. + /// Each 1 bit identifies a candidate by the bitfield bit index. + pub type CandidateBitfield = Bitfield; + /// A bitfield of core assignments. + pub type CoreBitfield = Bitfield; + + /// Errors that can occur when creating and manipulating bitfields. + #[derive(Debug)] + pub enum BitfieldError { + /// All bits are zero. + NullAssignment, + } + + /// A bit index in `Bitfield`. + #[cfg_attr(test, derive(PartialEq, Clone))] + pub struct BitIndex(pub usize); + + /// Helper trait to convert primitives to `BitIndex`. + pub trait AsBitIndex { + /// Returns the index of the corresponding bit in `Bitfield`. + fn as_bit_index(&self) -> BitIndex; + } + + impl Bitfield { + /// Returns the bit value at specified `index`. If `index` is greater than bitfield size, + /// returns `false`. + pub fn bit_at(&self, index: BitIndex) -> bool { + if self.0.len() <= index.0 { + false + } else { + self.0[index.0] + } + } + + /// Returns number of bits. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns the number of 1 bits. + pub fn count_ones(&self) -> usize { + self.0.count_ones() + } + + /// Returns the index of the first 1 bit. + pub fn first_one(&self) -> Option { + self.0.first_one() + } + + /// Returns an iterator over inner bits. + pub fn iter_ones(&self) -> bitvec::slice::IterOnes { + self.0.iter_ones() + } -impl UnsafeVRFOutput { - /// Get the slot. - pub fn slot(&self) -> Slot { - self.slot + /// For testing purpose, we want a inner mutable ref. + #[cfg(test)] + pub fn inner_mut(&mut self) -> &mut BitVec { + &mut self.0 + } + + /// Returns the inner bitfield and consumes `self`. + pub fn into_inner(self) -> BitVec { + self.0 + } } - /// Compute the randomness associated with this VRF output. - pub fn compute_randomness( - self, - authorities: &[(babe_primitives::AuthorityId, babe_primitives::BabeAuthorityWeight)], - randomness: &babe_primitives::Randomness, - epoch_index: u64, - ) -> Result { - let author = match authorities.get(self.authority_index as usize) { - None => return Err(ApprovalError::AuthorityOutOfBounds(self.authority_index as _)), - Some(x) => &x.0, - }; + impl AsBitIndex for CandidateIndex { + fn as_bit_index(&self) -> BitIndex { + BitIndex(*self as usize) + } + } - let pubkey = schnorrkel::PublicKey::from_bytes(author.as_slice()) - .map_err(ApprovalError::SchnorrkelSignature)?; + impl AsBitIndex for CoreIndex { + fn as_bit_index(&self) -> BitIndex { + BitIndex(self.0 as usize) + } + } - let transcript = sp_consensus_babe::make_vrf_transcript(randomness, self.slot, epoch_index); + impl AsBitIndex for usize { + fn as_bit_index(&self) -> BitIndex { + BitIndex(*self) + } + } - let inout = self - .vrf_output - .0 - .attach_input_hash(&pubkey, transcript.0) - .map_err(ApprovalError::SchnorrkelSignature)?; - Ok(RelayVRFStory(inout.make_bytes(RELAY_VRF_STORY_CONTEXT))) + impl From for Bitfield + where + T: AsBitIndex, + { + fn from(value: T) -> Self { + Self( + { + let mut bv = bitvec::bitvec![u8, Lsb0; 0; value.as_bit_index().0 + 1]; + bv.set(value.as_bit_index().0, true); + bv + }, + Default::default(), + ) + } + } + + impl TryFrom> for Bitfield + where + T: Into>, + { + type Error = BitfieldError; + + fn try_from(mut value: Vec) -> Result { + if value.is_empty() { + return Err(BitfieldError::NullAssignment) + } + + let initial_bitfield = + value.pop().expect("Just checked above it's not empty; qed").into(); + + Ok(Self( + value.into_iter().fold(initial_bitfield.0, |initial_bitfield, element| { + let mut bitfield: Bitfield = element.into(); + bitfield + .0 + .resize(std::cmp::max(initial_bitfield.len(), bitfield.0.len()), false); + bitfield.0.bitor(initial_bitfield) + }), + Default::default(), + )) + } + } + + /// Certificate is changed compared to `AssignmentCertKind`: + /// - introduced RelayVRFModuloCompact + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum AssignmentCertKindV2 { + /// Multiple assignment stories based on the VRF that authorized the relay-chain block + /// where the candidates were included. + /// + /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + #[codec(index = 0)] + RelayVRFModuloCompact { + /// A bitfield representing the core indices claimed by this assignment. + core_bitfield: CoreBitfield, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + #[codec(index = 1)] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + /// Deprectated assignment. Soon to be removed. + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + #[codec(index = 2)] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, + } + + /// A certification of assignment. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct AssignmentCertV2 { + /// The criterion which is claimed to be met by this cert. + pub kind: AssignmentCertKindV2, + /// The VRF showing the criterion is met. + pub vrf: VrfSignature, + } + + impl From for AssignmentCertV2 { + fn from(cert: super::v1::AssignmentCert) -> Self { + Self { + kind: match cert.kind { + super::v1::AssignmentCertKind::RelayVRFDelay { core_index } => + AssignmentCertKindV2::RelayVRFDelay { core_index }, + super::v1::AssignmentCertKind::RelayVRFModulo { sample } => + AssignmentCertKindV2::RelayVRFModulo { sample }, + }, + vrf: cert.vrf, + } + } } -} -/// Extract the slot number and relay VRF from a header. -/// -/// This fails if either there is no BABE `PreRuntime` digest or -/// the digest has type `SecondaryPlain`, which Substrate nodes do -/// not produce or accept anymore. -pub fn babe_unsafe_vrf_info(header: &Header) -> Option { - use babe_primitives::digests::CompatibleDigestItem; - - for digest in &header.digest.logs { - if let Some(pre) = digest.as_babe_pre_digest() { - let slot = pre.slot(); - let authority_index = pre.authority_index(); - - return pre.vrf_signature().map(|sig| UnsafeVRFOutput { - vrf_output: sig.output.clone(), - slot, - authority_index, + /// Errors that can occur when trying to convert to/from assignment v1/v2 + #[derive(Debug)] + pub enum AssignmentConversionError { + /// Assignment certificate is not supported in v1. + CertificateNotSupported, + } + + impl TryFrom for super::v1::AssignmentCert { + type Error = AssignmentConversionError; + fn try_from(cert: AssignmentCertV2) -> Result { + Ok(Self { + kind: match cert.kind { + AssignmentCertKindV2::RelayVRFDelay { core_index } => + super::v1::AssignmentCertKind::RelayVRFDelay { core_index }, + AssignmentCertKindV2::RelayVRFModulo { sample } => + super::v1::AssignmentCertKind::RelayVRFModulo { sample }, + // Not supported + _ => return Err(AssignmentConversionError::CertificateNotSupported), + }, + vrf: cert.vrf, }) } } - None + /// An assignment criterion which refers to the candidate under which the assignment is + /// relevant by block hash. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectAssignmentCertV2 { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The validator index. + pub validator: ValidatorIndex, + /// The cert itself. + pub cert: AssignmentCertV2, + } + + impl From for IndirectAssignmentCertV2 { + fn from(indirect_cert: super::v1::IndirectAssignmentCert) -> Self { + Self { + block_hash: indirect_cert.block_hash, + validator: indirect_cert.validator, + cert: indirect_cert.cert.into(), + } + } + } + + impl TryFrom for super::v1::IndirectAssignmentCert { + type Error = AssignmentConversionError; + fn try_from( + indirect_cert: IndirectAssignmentCertV2, + ) -> Result { + Ok(Self { + block_hash: indirect_cert.block_hash, + validator: indirect_cert.validator, + cert: indirect_cert.cert.try_into()?, + }) + } + } +} + +#[cfg(test)] +mod test { + use super::v2::{BitIndex, Bitfield}; + + use polkadot_primitives::{CandidateIndex, CoreIndex}; + + #[test] + fn test_assignment_bitfield_from_vec() { + let candidate_indices = vec![1u32, 7, 3, 10, 45, 8, 200, 2]; + let max_index = *candidate_indices.iter().max().unwrap(); + let bitfield = Bitfield::try_from(candidate_indices.clone()).unwrap(); + let candidate_indices = + candidate_indices.into_iter().map(|i| BitIndex(i as usize)).collect::>(); + + // Test 1 bits. + for index in candidate_indices.clone() { + assert!(bitfield.bit_at(index)); + } + + // Test 0 bits. + for index in 0..max_index { + if candidate_indices.contains(&BitIndex(index as usize)) { + continue + } + assert!(!bitfield.bit_at(BitIndex(index as usize))); + } + } + + #[test] + fn test_assignment_bitfield_invariant_msb() { + let core_indices = vec![CoreIndex(1), CoreIndex(3), CoreIndex(10), CoreIndex(20)]; + let mut bitfield = Bitfield::try_from(core_indices.clone()).unwrap(); + assert!(bitfield.inner_mut().pop().unwrap()); + + for i in 0..1024 { + assert!(Bitfield::try_from(CoreIndex(i)).unwrap().inner_mut().pop().unwrap()); + assert!(Bitfield::try_from(i).unwrap().inner_mut().pop().unwrap()); + } + } + + #[test] + fn test_assignment_bitfield_basic() { + let bitfield = Bitfield::try_from(CoreIndex(0)).unwrap(); + assert!(bitfield.bit_at(BitIndex(0))); + assert!(!bitfield.bit_at(BitIndex(1))); + assert_eq!(bitfield.len(), 1); + + let mut bitfield = Bitfield::try_from(20 as CandidateIndex).unwrap(); + assert!(bitfield.bit_at(BitIndex(20))); + assert_eq!(bitfield.inner_mut().count_ones(), 1); + assert_eq!(bitfield.len(), 21); + } } diff --git a/polkadot/node/service/src/parachains_db/mod.rs b/polkadot/node/service/src/parachains_db/mod.rs index 519afbe0ccd1..92f3f167f22f 100644 --- a/polkadot/node/service/src/parachains_db/mod.rs +++ b/polkadot/node/service/src/parachains_db/mod.rs @@ -41,7 +41,15 @@ pub(crate) mod columns { pub const COL_SESSION_WINDOW_DATA: u32 = 5; } + // Version 4 only changed structures in approval voting, so we can re-export the v4 definitions. pub mod v3 { + pub use super::v4::{ + COL_APPROVAL_DATA, COL_AVAILABILITY_DATA, COL_AVAILABILITY_META, + COL_CHAIN_SELECTION_DATA, COL_DISPUTE_COORDINATOR_DATA, NUM_COLUMNS, ORDERED_COL, + }; + } + + pub mod v4 { pub const NUM_COLUMNS: u32 = 5; pub const COL_AVAILABILITY_DATA: u32 = 0; pub const COL_AVAILABILITY_META: u32 = 1; @@ -73,14 +81,14 @@ pub struct ColumnsConfig { /// The real columns used by the parachains DB. #[cfg(any(test, feature = "full-node"))] pub const REAL_COLUMNS: ColumnsConfig = ColumnsConfig { - col_availability_data: columns::v3::COL_AVAILABILITY_DATA, - col_availability_meta: columns::v3::COL_AVAILABILITY_META, - col_approval_data: columns::v3::COL_APPROVAL_DATA, - col_chain_selection_data: columns::v3::COL_CHAIN_SELECTION_DATA, - col_dispute_coordinator_data: columns::v3::COL_DISPUTE_COORDINATOR_DATA, + col_availability_data: columns::v4::COL_AVAILABILITY_DATA, + col_availability_meta: columns::v4::COL_AVAILABILITY_META, + col_approval_data: columns::v4::COL_APPROVAL_DATA, + col_chain_selection_data: columns::v4::COL_CHAIN_SELECTION_DATA, + col_dispute_coordinator_data: columns::v4::COL_DISPUTE_COORDINATOR_DATA, }; -#[derive(PartialEq)] +#[derive(PartialEq, Copy, Clone)] pub(crate) enum DatabaseKind { ParityDB, RocksDB, @@ -125,28 +133,28 @@ pub fn open_creating_rocksdb( let path = root.join("parachains").join("db"); - let mut db_config = DatabaseConfig::with_columns(columns::v3::NUM_COLUMNS); + let mut db_config = DatabaseConfig::with_columns(columns::v4::NUM_COLUMNS); let _ = db_config .memory_budget - .insert(columns::v3::COL_AVAILABILITY_DATA, cache_sizes.availability_data); + .insert(columns::v4::COL_AVAILABILITY_DATA, cache_sizes.availability_data); let _ = db_config .memory_budget - .insert(columns::v3::COL_AVAILABILITY_META, cache_sizes.availability_meta); + .insert(columns::v4::COL_AVAILABILITY_META, cache_sizes.availability_meta); let _ = db_config .memory_budget - .insert(columns::v3::COL_APPROVAL_DATA, cache_sizes.approval_data); + .insert(columns::v4::COL_APPROVAL_DATA, cache_sizes.approval_data); let path_str = path .to_str() .ok_or_else(|| other_io_error(format!("Bad database path: {:?}", path)))?; std::fs::create_dir_all(&path_str)?; - upgrade::try_upgrade_db(&path, DatabaseKind::RocksDB)?; + upgrade::try_upgrade_db(&path, DatabaseKind::RocksDB, upgrade::CURRENT_VERSION)?; let db = Database::open(&db_config, &path_str)?; let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new( db, - columns::v3::ORDERED_COL, + columns::v4::ORDERED_COL, ); Ok(Arc::new(db)) @@ -164,14 +172,14 @@ pub fn open_creating_paritydb( .ok_or_else(|| other_io_error(format!("Bad database path: {:?}", path)))?; std::fs::create_dir_all(&path_str)?; - upgrade::try_upgrade_db(&path, DatabaseKind::ParityDB)?; + upgrade::try_upgrade_db(&path, DatabaseKind::ParityDB, upgrade::CURRENT_VERSION)?; let db = parity_db::Db::open_or_create(&upgrade::paritydb_version_3_config(&path)) .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?; let db = polkadot_node_subsystem_util::database::paritydb_impl::DbAdapter::new( db, - columns::v3::ORDERED_COL, + columns::v4::ORDERED_COL, ); Ok(Arc::new(db)) } diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index 54ef97afd71c..b99f885176b0 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -22,13 +22,17 @@ use std::{ str::FromStr, }; +use polkadot_node_core_approval_voting::approval_db::v2::{ + migration_helpers::v1_to_v2, Config as ApprovalDbConfig, +}; type Version = u32; /// Version file name. const VERSION_FILE_NAME: &'static str = "parachain_db_version"; /// Current db version. -const CURRENT_VERSION: Version = 3; +/// Version 4 changes approval db format for `OurAssignment`. +pub(crate) const CURRENT_VERSION: Version = 4; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -38,6 +42,10 @@ pub enum Error { CorruptedVersionFile, #[error("Parachains DB has a future version (expected {current:?}, found {got:?})")] FutureVersion { current: Version, got: Version }, + #[error("Parachain DB migration failed")] + MigrationFailed, + #[error("Parachain DB migration would take forever")] + MigrationLoop, } impl From for io::Error { @@ -49,10 +57,42 @@ impl From for io::Error { } } -/// Try upgrading parachain's database to the current version. -pub(crate) fn try_upgrade_db(db_path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +/// Try upgrading parachain's database to a target version. +pub(crate) fn try_upgrade_db( + db_path: &Path, + db_kind: DatabaseKind, + target_version: Version, +) -> Result<(), Error> { + // Ensure we don't loop forever below befcause of a bug. + const MAX_MIGRATIONS: u32 = 30; + + #[cfg(test)] + remove_file_lock(&db_path); + + // Loop migrations until we reach the target version. + for _ in 0..MAX_MIGRATIONS { + let version = try_upgrade_db_to_next_version(db_path, db_kind)?; + + #[cfg(test)] + remove_file_lock(&db_path); + + if version == target_version { + return Ok(()) + } + } + + Err(Error::MigrationLoop) +} + +/// Try upgrading parachain's database to the next version. +/// If successfull, it returns the current version. +pub(crate) fn try_upgrade_db_to_next_version( + db_path: &Path, + db_kind: DatabaseKind, +) -> Result { let is_empty = db_path.read_dir().map_or(true, |mut d| d.next().is_none()); - if !is_empty { + + let new_version = if !is_empty { match get_db_version(db_path)? { // 0 -> 1 migration Some(0) => migrate_from_version_0_to_1(db_path, db_kind)?, @@ -60,21 +100,26 @@ pub(crate) fn try_upgrade_db(db_path: &Path, db_kind: DatabaseKind) -> Result<() Some(1) => migrate_from_version_1_to_2(db_path, db_kind)?, // 2 -> 3 migration Some(2) => migrate_from_version_2_to_3(db_path, db_kind)?, + // 3 -> 4 migration + Some(3) => migrate_from_version_3_to_4(db_path, db_kind)?, // Already at current version, do nothing. - Some(CURRENT_VERSION) => (), + Some(CURRENT_VERSION) => CURRENT_VERSION, // This is an arbitrary future version, we don't handle it. Some(v) => return Err(Error::FutureVersion { current: CURRENT_VERSION, got: v }), // No version file. For `RocksDB` we dont need to do anything. - None if db_kind == DatabaseKind::RocksDB => (), + None if db_kind == DatabaseKind::RocksDB => CURRENT_VERSION, // No version file. `ParityDB` did not previously have a version defined. // We handle this as a `0 -> 1` migration. None if db_kind == DatabaseKind::ParityDB => migrate_from_version_0_to_1(db_path, db_kind)?, None => unreachable!(), } - } + } else { + CURRENT_VERSION + }; - update_version(db_path) + update_version(db_path, new_version)?; + Ok(new_version) } /// Reads current database version from the file at given path. @@ -91,9 +136,9 @@ fn get_db_version(path: &Path) -> Result, Error> { /// Writes current database version to the file. /// Creates a new file if the version file does not exist yet. -fn update_version(path: &Path) -> Result<(), Error> { +fn update_version(path: &Path, new_version: Version) -> Result<(), Error> { fs::create_dir_all(path)?; - fs::write(version_file_path(path), CURRENT_VERSION.to_string()).map_err(Into::into) + fs::write(version_file_path(path), new_version.to_string()).map_err(Into::into) } /// Returns the version file path. @@ -103,7 +148,7 @@ fn version_file_path(path: &Path) -> PathBuf { file_path } -fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 0 to version 1 ..."); match db_kind { @@ -116,7 +161,7 @@ fn migrate_from_version_0_to_1(path: &Path, db_kind: DatabaseKind) -> Result<(), }) } -fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 1 to version 2 ..."); match db_kind { @@ -129,7 +174,48 @@ fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result<(), }) } -fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result<(), Error> { +// Migrade approval voting database. `OurAssignment` has been changed to support the v2 assignments. +// As these are backwards compatible, we'll convert the old entries in the new format. +fn migrate_from_version_3_to_4(path: &Path, db_kind: DatabaseKind) -> Result { + gum::info!(target: LOG_TARGET, "Migrating parachains db from version 3 to version 4 ..."); + use polkadot_node_subsystem_util::database::{ + kvdb_impl::DbAdapter as RocksDbAdapter, paritydb_impl::DbAdapter as ParityDbAdapter, + }; + use std::sync::Arc; + + let approval_db_config = + ApprovalDbConfig { col_approval_data: super::REAL_COLUMNS.col_approval_data }; + + let _result = match db_kind { + DatabaseKind::ParityDB => { + let db = ParityDbAdapter::new( + parity_db::Db::open(&paritydb_version_3_config(path)) + .map_err(|e| other_io_error(format!("Error opening db {:?}", e)))?, + super::columns::v3::ORDERED_COL, + ); + + v1_to_v2(Arc::new(db), approval_db_config).map_err(|_| Error::MigrationFailed)?; + }, + DatabaseKind::RocksDB => { + let db_path = path + .to_str() + .ok_or_else(|| super::other_io_error("Invalid database path".into()))?; + let db_cfg = + kvdb_rocksdb::DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); + let db = RocksDbAdapter::new( + kvdb_rocksdb::Database::open(&db_cfg, db_path)?, + &super::columns::v3::ORDERED_COL, + ); + + v1_to_v2(Arc::new(db), approval_db_config).map_err(|_| Error::MigrationFailed)?; + }, + }; + + gum::info!(target: LOG_TARGET, "Migration complete! "); + Ok(CURRENT_VERSION) +} + +fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result { gum::info!(target: LOG_TARGET, "Migrating parachains db from version 2 to version 3 ..."); match db_kind { DatabaseKind::ParityDB => paritydb_migrate_from_version_2_to_3(path), @@ -143,7 +229,7 @@ fn migrate_from_version_2_to_3(path: &Path, db_kind: DatabaseKind) -> Result<(), /// Migration from version 0 to version 1: /// * the number of columns has changed from 3 to 5; -fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -155,12 +241,12 @@ fn rocksdb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { db.add_column()?; db.add_column()?; - Ok(()) + Ok(1) } /// Migration from version 1 to version 2: /// * the number of columns has changed from 5 to 6; -fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -171,10 +257,10 @@ fn rocksdb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { db.add_column()?; - Ok(()) + Ok(2) } -fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { +fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result { use kvdb_rocksdb::{Database, DatabaseConfig}; let db_path = path @@ -185,7 +271,7 @@ fn rocksdb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { db.remove_last_column()?; - Ok(()) + Ok(3) } // This currently clears columns which had their configs altered between versions. @@ -249,7 +335,7 @@ fn paritydb_fix_columns( pub(crate) fn paritydb_version_1_config(path: &Path) -> parity_db::Options { let mut options = parity_db::Options::with_columns(&path, super::columns::v1::NUM_COLUMNS as u8); - for i in columns::v3::ORDERED_COL { + for i in columns::v4::ORDERED_COL { options.columns[*i as usize].btree_index = true; } @@ -260,7 +346,7 @@ pub(crate) fn paritydb_version_1_config(path: &Path) -> parity_db::Options { pub(crate) fn paritydb_version_2_config(path: &Path) -> parity_db::Options { let mut options = parity_db::Options::with_columns(&path, super::columns::v2::NUM_COLUMNS as u8); - for i in columns::v3::ORDERED_COL { + for i in columns::v4::ORDERED_COL { options.columns[*i as usize].btree_index = true; } @@ -278,48 +364,160 @@ pub(crate) fn paritydb_version_3_config(path: &Path) -> parity_db::Options { options } +/// Database configuration for version 0. This is useful just for testing. +#[cfg(test)] +pub(crate) fn paritydb_version_0_config(path: &Path) -> parity_db::Options { + let mut options = + parity_db::Options::with_columns(&path, super::columns::v0::NUM_COLUMNS as u8); + options.columns[super::columns::v4::COL_AVAILABILITY_META as usize].btree_index = true; + + options +} + /// Migration from version 0 to version 1. /// Cases covered: /// - upgrading from v0.9.23 or earlier -> the `dispute coordinator column` was changed /// - upgrading from v0.9.24+ -> this is a no op assuming the DB has been manually fixed as per /// release notes -fn paritydb_migrate_from_version_0_to_1(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_0_to_1(path: &Path) -> Result { // Delete the `dispute coordinator` column if needed (if column configuration is changed). paritydb_fix_columns( path, paritydb_version_1_config(path), - vec![super::columns::v3::COL_DISPUTE_COORDINATOR_DATA], + vec![super::columns::v4::COL_DISPUTE_COORDINATOR_DATA], )?; - Ok(()) + Ok(1) } /// Migration from version 1 to version 2: /// - add a new column for session information storage -fn paritydb_migrate_from_version_1_to_2(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_1_to_2(path: &Path) -> Result { let mut options = paritydb_version_1_config(path); // Adds the session info column. parity_db::Db::add_column(&mut options, Default::default()) .map_err(|e| other_io_error(format!("Error adding column {:?}", e)))?; - Ok(()) + Ok(2) } /// Migration from version 2 to version 3: /// - drop the column used by `RollingSessionWindow` -fn paritydb_migrate_from_version_2_to_3(path: &Path) -> Result<(), Error> { +fn paritydb_migrate_from_version_2_to_3(path: &Path) -> Result { parity_db::Db::drop_last_column(&mut paritydb_version_2_config(path)) .map_err(|e| other_io_error(format!("Error removing COL_SESSION_WINDOW_DATA {:?}", e)))?; - Ok(()) + Ok(3) +} + +/// Remove the lock file. If file is locked, it will wait up to 1s. +#[cfg(test)] +pub fn remove_file_lock(path: &std::path::Path) { + use std::{io::ErrorKind, thread::sleep, time::Duration}; + + let mut lock_path = std::path::PathBuf::from(path); + lock_path.push("lock"); + + for _ in 0..10 { + let result = std::fs::remove_file(lock_path.as_path()); + match result { + Err(error) => match error.kind() { + ErrorKind::WouldBlock => { + sleep(Duration::from_millis(100)); + continue + }, + _ => return, + }, + Ok(_) => {}, + } + } + + unreachable!("Database is locked, waited 1s for lock file: {:?}", lock_path); } #[cfg(test)] mod tests { use super::{ - columns::{v2::COL_SESSION_WINDOW_DATA, v3::*}, + columns::{v2::COL_SESSION_WINDOW_DATA, v4::*}, *, }; + use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_fill_test_data; + + #[test] + fn test_paritydb_migrate_0_to_1() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + { + let db = Db::open_or_create(&paritydb_version_0_config(&path)).unwrap(); + + db.commit(vec![( + COL_AVAILABILITY_META as u8, + b"5678".to_vec(), + Some(b"somevalue".to_vec()), + )]) + .unwrap(); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 1).unwrap(); + + let db = Db::open(&paritydb_version_1_config(&path)).unwrap(); + assert_eq!( + db.get(COL_AVAILABILITY_META as u8, b"5678").unwrap(), + Some("somevalue".as_bytes().to_vec()) + ); + } + + #[test] + fn test_paritydb_migrate_1_to_2() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "1").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_1_config(&path)).unwrap(); + + // Write some dummy data + db.commit(vec![( + COL_DISPUTE_COORDINATOR_DATA as u8, + b"1234".to_vec(), + Some(b"somevalue".to_vec()), + )]) + .unwrap(); + + assert_eq!(db.num_columns(), columns::v1::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 2).unwrap(); + + let db = Db::open(&paritydb_version_2_config(&path)).unwrap(); + + assert_eq!(db.num_columns(), columns::v2::NUM_COLUMNS as u8); + + assert_eq!( + db.get(COL_DISPUTE_COORDINATOR_DATA as u8, b"1234").unwrap(), + Some("somevalue".as_bytes().to_vec()) + ); + + // Test we can write the new column. + db.commit(vec![( + COL_SESSION_WINDOW_DATA as u8, + b"1337".to_vec(), + Some(b"0xdeadb00b".to_vec()), + )]) + .unwrap(); + + // Read back data from new column. + assert_eq!( + db.get(COL_SESSION_WINDOW_DATA as u8, b"1337").unwrap(), + Some("0xdeadb00b".as_bytes().to_vec()) + ); + } #[test] fn test_rocksdb_migrate_1_to_2() { @@ -338,7 +536,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "1").expect("Failed to write DB version"); { - let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); db.write(DBTransaction { ops: vec![DBOp::Insert { col: COL_DISPUTE_COORDINATOR_DATA, @@ -349,14 +547,14 @@ mod tests { .unwrap(); } - try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB).unwrap(); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 2).unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v2::NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v2::NUM_COLUMNS); - let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); assert_eq!( db.get(COL_DISPUTE_COORDINATOR_DATA, b"1234").unwrap(), @@ -380,6 +578,108 @@ mod tests { ); } + #[test] + fn test_migrate_3_to_4() { + use kvdb_rocksdb::{Database, DatabaseConfig}; + use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_sanity_check; + use polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter; + + let db_dir = tempfile::tempdir().unwrap(); + let db_path = db_dir.path().to_str().unwrap(); + let db_cfg: DatabaseConfig = DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); + + let approval_cfg = ApprovalDbConfig { + col_approval_data: crate::parachains_db::REAL_COLUMNS.col_approval_data, + }; + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(db_dir.path()), "3").expect("Failed to write DB version"); + let expected_candidates = { + let db = Database::open(&db_cfg, db_path.clone()).unwrap(); + assert_eq!(db.num_columns(), super::columns::v3::NUM_COLUMNS as u32); + let db = DbAdapter::new(db, columns::v3::ORDERED_COL); + // Fill the approval voting column with test data. + v1_to_v2_fill_test_data(std::sync::Arc::new(db), approval_cfg).unwrap() + }; + + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); + + let db_cfg = DatabaseConfig::with_columns(super::columns::v4::NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).unwrap(); + let db = DbAdapter::new(db, columns::v4::ORDERED_COL); + + v1_to_v2_sanity_check(std::sync::Arc::new(db), approval_cfg, expected_candidates).unwrap(); + } + + #[test] + fn test_rocksdb_migrate_0_to_4() { + use kvdb_rocksdb::{Database, DatabaseConfig}; + + let db_dir = tempfile::tempdir().unwrap(); + let db_path = db_dir.path().to_str().unwrap(); + + fs::write(version_file_path(db_dir.path()), "0").expect("Failed to write DB version"); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); + + let db_cfg = DatabaseConfig::with_columns(super::columns::v4::NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).unwrap(); + + assert_eq!(db.num_columns(), columns::v4::NUM_COLUMNS); + } + + #[test] + fn test_paritydb_migrate_0_to_4() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "0").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_0_config(&path)).unwrap(); + assert_eq!(db.num_columns(), columns::v0::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 4).unwrap(); + + let db = Db::open(&paritydb_version_3_config(&path)).unwrap(); + assert_eq!(db.num_columns(), columns::v4::NUM_COLUMNS as u8); + } + + #[test] + fn test_paritydb_migrate_2_to_3() { + use parity_db::Db; + + let db_dir = tempfile::tempdir().unwrap(); + let path = db_dir.path(); + let test_key = b"1337"; + + // We need to properly set db version for upgrade to work. + fs::write(version_file_path(path), "2").expect("Failed to write DB version"); + + { + let db = Db::open_or_create(&paritydb_version_2_config(&path)).unwrap(); + + // Write some dummy data + db.commit(vec![( + COL_SESSION_WINDOW_DATA as u8, + test_key.to_vec(), + Some(b"0xdeadb00b".to_vec()), + )]) + .unwrap(); + + assert_eq!(db.num_columns(), columns::v2::NUM_COLUMNS as u8); + } + + try_upgrade_db(&path, DatabaseKind::ParityDB, 3).unwrap(); + + let db = Db::open(&paritydb_version_3_config(&path)).unwrap(); + + assert_eq!(db.num_columns(), columns::v3::NUM_COLUMNS as u8); + } + #[test] fn test_rocksdb_migrate_2_to_3() { use kvdb_rocksdb::{Database, DatabaseConfig}; @@ -387,6 +687,7 @@ mod tests { let db_dir = tempfile::tempdir().unwrap(); let db_path = db_dir.path().to_str().unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v2::NUM_COLUMNS); + { let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v2::NUM_COLUMNS as u32); @@ -395,7 +696,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "2").expect("Failed to write DB version"); - try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB).unwrap(); + try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 3).unwrap(); let db_cfg = DatabaseConfig::with_columns(super::columns::v3::NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).unwrap(); diff --git a/polkadot/node/service/src/tests.rs b/polkadot/node/service/src/tests.rs index 86119662d9bc..26c8083185d8 100644 --- a/polkadot/node/service/src/tests.rs +++ b/polkadot/node/service/src/tests.rs @@ -17,7 +17,7 @@ use super::{relay_chain_selection::*, *}; use futures::channel::oneshot::Receiver; -use polkadot_node_primitives::approval::VrfSignature; +use polkadot_node_primitives::approval::v2::VrfSignature; use polkadot_node_subsystem::messages::{AllMessages, BlockDescription}; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; diff --git a/polkadot/node/subsystem-types/Cargo.toml b/polkadot/node/subsystem-types/Cargo.toml index 4f2d753b7993..51337c2cbeb3 100644 --- a/polkadot/node/subsystem-types/Cargo.toml +++ b/polkadot/node/subsystem-types/Cargo.toml @@ -24,3 +24,4 @@ smallvec = "1.8.0" substrate-prometheus-endpoint = { path = "../../../substrate/utils/prometheus" } thiserror = "1.0.31" async-trait = "0.1.57" +bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 8adc39eed56d..f561a0e28512 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -32,7 +32,10 @@ use polkadot_node_network_protocol::{ self as net_protocol, peer_set::PeerSet, request_response::Requests, PeerId, }; use polkadot_node_primitives::{ - approval::{BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote}, + approval::{ + v1::{BlockApprovalMeta, IndirectSignedApprovalVote}, + v2::{CandidateBitfield, IndirectAssignmentCertV2}, + }, AvailableData, BabeEpoch, BlockWeight, CandidateVotes, CollationGenerationConfig, CollationSecondedSignal, DisputeMessage, DisputeStatus, ErasureChunk, PoV, SignedDisputeStatement, SignedFullStatement, SignedFullStatementWithPVD, SubmitCollationParams, @@ -833,6 +836,8 @@ pub enum AssignmentCheckError { InvalidCert(ValidatorIndex, String), #[error("Internal state mismatch: {0:?}, {1:?}")] Internal(Hash, CandidateHash), + #[error("Oversized candidate or core bitfield >= {0}")] + InvalidBitfield(usize), } /// The result type of [`ApprovalVotingMessage::CheckAndImportApproval`] request. @@ -898,8 +903,8 @@ pub enum ApprovalVotingMessage { /// Check if the assignment is valid and can be accepted by our view of the protocol. /// Should not be sent unless the block hash is known. CheckAndImportAssignment( - IndirectAssignmentCert, - CandidateIndex, + IndirectAssignmentCertV2, + CandidateBitfield, oneshot::Sender, ), /// Check if the approval vote is valid and can be accepted by our view of the @@ -934,7 +939,7 @@ pub enum ApprovalDistributionMessage { NewBlocks(Vec), /// Distribute an assignment cert from the local validator. The cert is assumed /// to be valid, relevant, and for the given relay-parent and validator index. - DistributeAssignment(IndirectAssignmentCert, CandidateIndex), + DistributeAssignment(IndirectAssignmentCertV2, CandidateBitfield), /// Distribute an approval vote for the local validator. The approval vote is assumed to be /// valid, relevant, and the corresponding approval already issued. /// If not, the subsystem is free to drop the message. diff --git a/polkadot/node/subsystem-util/src/reputation.rs b/polkadot/node/subsystem-util/src/reputation.rs index 89e3eb64df9b..35746dd5fef3 100644 --- a/polkadot/node/subsystem-util/src/reputation.rs +++ b/polkadot/node/subsystem-util/src/reputation.rs @@ -25,6 +25,7 @@ use std::{collections::HashMap, time::Duration}; /// Default delay for sending reputation changes pub const REPUTATION_CHANGE_INTERVAL: Duration = Duration::from_secs(30); +const LOG_TARGET: &'static str = "parachain::reputation-aggregator"; type BatchReputationChange = HashMap; @@ -75,6 +76,10 @@ impl ReputationAggregator { peer_id: PeerId, rep: UnifiedReputationChange, ) { + if rep.cost_or_benefit() < 0 { + gum::debug!(target: LOG_TARGET, peer = ?peer_id, ?rep, "Modify reputation"); + } + if (self.send_immediately_if)(rep) { self.single_send(sender, peer_id, rep).await; } else { diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index 88744e50cf79..375b8f1f12b3 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -49,22 +49,27 @@ struct TrancheEntry { assignments: Vec<(ValidatorIndex, Tick)>, } -struct OurAssignment { - cert: AssignmentCert, - tranche: DelayTranche, - validator_index: ValidatorIndex, - triggered: bool, +pub struct OurAssignment { + /// Our assignment certificate. + cert: AssignmentCertV2, + /// The tranche for which the assignment refers to. + tranche: DelayTranche, + /// Our validator index for the session in which the candidates were included. + validator_index: ValidatorIndex, + /// Whether the assignment has been triggered already. + triggered: bool, } -struct ApprovalEntry { - tranches: Vec, // sorted ascending by tranche number. - backing_group: GroupIndex, - our_assignment: Option, - our_approval_sig: Option, - assignments: Bitfield, // n_validators bits - approved: bool, +pub struct ApprovalEntry { + tranches: Vec, // sorted ascending by tranche number. + backing_group: GroupIndex, + our_assignment: Option, + our_approval_sig: Option, + assigned_validators: Bitfield, // `n_validators` bits. + approved: bool, } + struct CandidateEntry { candidate: CandidateReceipt, session: SessionIndex, @@ -200,6 +205,8 @@ On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we che * Determine the claimed core index by looking up the candidate with given index in `block_entry.candidates`. Return `AssignmentCheckResult::Bad` if missing. * Check the assignment cert * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ sample.encode()` as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We set `core_index = vrf.make_bytes().to_u32() % session_info.n_cores`. If the `BlockEntry` causes inclusion of a candidate at `core_index`, then this is a valid assignment for the candidate at `core_index` and has delay tranche 0. Otherwise, it can be ignored. + * If the cert kind is `RelayVRFModuloCompact`, then the certificate is valid as long as the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ relay_vrf_samples.encode()` as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We enforce that all `core_bitfield` indices are included in the set of the core indices sampled from the VRF Output. The assignment is considered a valid tranche0 assignment for all claimed candidates if all `core_bitfield` indices match the core indices where the claimed candidates were included at. + * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not cause inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included candidate. The delay tranche for the assignment is determined by reducing `(vrf.make_bytes().to_u64() % (session_info.n_delay_tranches + session_info.zeroth_delay_tranche_width)).saturating_sub(session_info.zeroth_delay_tranche_width)`. * We also check that the core index derived by the output is covered by the `VRFProof` by means of an auxiliary signature. * If the delay tranche is too far in the future, return `AssignmentCheckResult::TooFarInFuture`. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index 693822ce0797..aa513c16292d 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -98,12 +98,14 @@ We want checkers for candidate equivocations that lie outside our preferred rela Assignment criteria compute actual assignments using stories and the validators' secret approval assignment key. Assignment criteria output a `Position` consisting of both a `ParaId` to be checked, as well as a precedence `DelayTranche` for when the assignment becomes valid. -Assignment criteria come in three flavors, `RelayVRFModulo`, `RelayVRFDelay` and `RelayEquivocation`. Among these, both `RelayVRFModulo` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. +Assignment criteria come in four flavors, `RelayVRFModuloCompact`, `RelayVRFDelay`, `RelayEquivocation` and the deprecated `RelayVRFModulo`. Among these, `RelayVRFModulo`, `RelayVRFModuloCompact` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. Among these, we have two distinct VRF output computations: `RelayVRFModulo` runs several distinct samples whose VRF input is the `RelayVRFStory` and the sample number. It computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core", reduces this number modulo the number of availability cores, and outputs the candidate just declared available by, and included by aka leaving, that availability core. We drop any samples that return no candidate because no candidate was leaving the sampled availability core in this relay chain block. We choose three samples initially, but we could make polkadot more secure and efficient by increasing this to four or five, and reducing the backing checks accordingly. All successful `RelayVRFModulo` samples are assigned delay tranche zero. +`RelayVRFModuloCompact` runs a single samples whose VRF input is the `RelayVRFStory` and the sample count. Similar to `RelayVRFModulo` introduces multiple core assignments for tranche zero. It computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core v2" and samples up to 160 bytes of the output as an array of `u32`. Then reduces each `u32` modulo the number of availability cores, and outputs up to `relay_vrf_modulo_samples` availability core indices. + There is no sampling process for `RelayVRFDelay` and `RelayEquivocation`. We instead run them on specific candidates and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared available by, a relay chain block, and inputs the associated VRF output via `RelayVRFStory`. `RelayEquivocation` runs only on candidate block equivocations, and inputs their block hashes via the `RelayEquivocation` story. `RelayVRFDelay` and `RelayEquivocation` both compute their output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Tranche" and reduce the result modulo `num_delay_tranches + zeroth_delay_tranche_width`, and consolidate results 0 through `zeroth_delay_tranche_width` to be 0. In this way, they ensure the zeroth delay tranche has `zeroth_delay_tranche_width+1` times as many assignments as any other tranche. @@ -114,9 +116,9 @@ As future work (or TODO?), we should merge assignment notices with the same dela We track all validators' announced approval assignments for each candidate associated to each relay chain block, which tells us which validators were assigned to which candidates. -We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. +We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo/RelayVRFModuloCompact` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. -We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` assignments. +We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` and `RelayVRFModuloCompact` assignments. In other words, if some candidate `C` needs more approval checkers by the time we reach round `t` then any validators with an assignment to `C` in delay tranche `t` gossip their send assignment notice for `C`, and begin reconstruction and validation for 'C. If however `C` reached enough assignments, then validators with later assignments skip announcing their assignments. @@ -164,7 +166,7 @@ We need the chain to win in this case, but doing this requires imposing an annoy ## Parameters -We prefer doing approval checkers assignments under `RelayVRFModulo` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` although assignment levels have never been properly analyzed. +We prefer doing approval checkers assignments under `RelayVRFModulo` or `RelayVRFModuloCompact` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` or `RelayVRFModuloCompact` although assignment levels have never been properly analyzed. Our delay criteria `RelayVRFDelay` and `RelayEquivocation` both have two primary paramaters, expected checkers per tranche and the zeroth delay tranche width. diff --git a/polkadot/roadmap/implementers-guide/src/types/approval.md b/polkadot/roadmap/implementers-guide/src/types/approval.md index b58e0a8187e1..2cc9b34cc700 100644 --- a/polkadot/roadmap/implementers-guide/src/types/approval.md +++ b/polkadot/roadmap/implementers-guide/src/types/approval.md @@ -20,6 +20,35 @@ enum AssignmentCertKind { } } +enum AssignmentCertKindV2 { + /// Multiple assignment stories based on the VRF that authorized the relay-chain block where the + /// candidates were included. + /// + /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModuloCompact { + /// A bitfield representing the core indices claimed by this assignment. + core_bitfield: CoreBitfield, + }, + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with the index of a particular core. + /// + /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + RelayVRFDelay { + /// The core index chosen in this cert. + core_index: CoreIndex, + }, + /// Deprectated assignment. Soon to be removed. + /// + /// An assignment story based on the VRF that authorized the relay-chain block where the + /// candidate was included combined with a sample number. + /// + /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + RelayVRFModulo { + /// The sample number used in this cert. + sample: u32, + }, +} + struct AssignmentCert { // The criterion which is claimed to be met by this cert. kind: AssignmentCertKind, diff --git a/polkadot/roadmap/implementers-guide/src/types/runtime.md b/polkadot/roadmap/implementers-guide/src/types/runtime.md index 79da899bd35e..4b97409f8df3 100644 --- a/polkadot/roadmap/implementers-guide/src/types/runtime.md +++ b/polkadot/roadmap/implementers-guide/src/types/runtime.md @@ -55,7 +55,7 @@ struct HostConfiguration { pub zeroth_delay_tranche_width: u32, /// The number of validators needed to approve a block. pub needed_approvals: u32, - /// The number of samples to do of the RelayVRFModulo approval assignment criterion. + /// The number of samples to use in `RelayVRFModulo` or `RelayVRFModuloCompact` approval assignment criterions. pub relay_vrf_modulo_samples: u32, /// Total number of individual messages allowed in the parachain -> relay-chain message queue. pub max_upward_queue_count: u32, diff --git a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml index 62e081d1de0e..1c8df44cbaa7 100644 --- a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml +++ b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml @@ -160,6 +160,34 @@ zombienet-tests-parachains-disputes-past-session: tags: - zombienet-polkadot-integration-test +zombienet-tests-parachains-max-tranche0-approvals: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + variables: + RUN_IN_CONTAINER: "1" + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0005-parachains-max-tranche0.zndsl" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test zombienet-test-parachains-upgrade-smoke-test: stage: zombienet @@ -182,7 +210,7 @@ zombienet-test-parachains-upgrade-smoke-test: - echo "${GH_DIR}" - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image + - export COL_IMAGE="docker.io/parity/polkadot-parachain:latest" # Use cumulus lastest image script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" diff --git a/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.toml b/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.toml new file mode 100644 index 000000000000..ab7fa0195d13 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.toml @@ -0,0 +1,40 @@ +[settings] +timeout = 1000 +bootnode = true + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] + max_validators_per_core = 1 + needed_approvals = 7 + relay_vrf_modulo_samples = 5 + +[relaychain] +default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +chain = "rococo-local" +chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" +default_command = "polkadot" + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.node_groups]] + name = "some-validator" + count = 8 + args = ["-lparachain=debug,runtime=debug"] + +{% for id in range(2000,2005) %} +[[parachains]] +id = {{id}} +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size={{10000*(id-1999)}} --pvf-complexity={{id - 1999}}" + [parachains.collator] + image = "{{COL_IMAGE}}" + name = "collator" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size={{10000*(id-1999)}}", "--parachain-id={{id}}", "--pvf-complexity={{id - 1999}}"] +{% endfor %} + +[types.Header] +number = "u64" +parent_hash = "Hash" +post_state = "Hash" diff --git a/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl b/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl new file mode 100644 index 000000000000..48a56f60c9bc --- /dev/null +++ b/polkadot/zombienet_tests/functional/0005-parachains-max-tranche0.zndsl @@ -0,0 +1,27 @@ +Description: Test if parachains make progress with most of approvals being tranch0 +Network: ./0005-parachains-max-tranche0.toml +Creds: config + +# Check authority status. +some-validator-0: reports node_roles is 4 +some-validator-1: reports node_roles is 4 +some-validator-3: reports node_roles is 4 +some-validator-4: reports node_roles is 4 +some-validator-5: reports node_roles is 4 +some-validator-6: reports node_roles is 4 +some-validator-7: reports node_roles is 4 + +some-validator-0: parachain 2000 block height is at least 5 within 180 seconds +some-validator-1: parachain 2001 block height is at least 5 within 180 seconds +some-validator-2: parachain 2002 block height is at least 5 within 180 seconds +some-validator-3: parachain 2003 block height is at least 5 within 180 seconds +some-validator-4: parachain 2004 block height is at least 5 within 180 seconds + +some-validator-0: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-1: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-2: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-3: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-4: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-5: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-6: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 +some-validator-7: reports polkadot_parachain_approval_checking_finality_lag is lower than 2 diff --git a/polkadot/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml b/polkadot/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml index 0becb408550a..88b789f37fa1 100644 --- a/polkadot/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml +++ b/polkadot/zombienet_tests/smoke/0002-parachains-upgrade-smoke-test.toml @@ -31,7 +31,7 @@ cumulus_based = true [parachains.collator] name = "collator01" image = "{{COL_IMAGE}}" - command = "polkadot-collator" + command = "polkadot-parachain" [[parachains.collator.env]] name = "RUST_LOG" From d04c18239763fb50f29a49ddd789b7ed242a98d2 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 25 Aug 2023 19:15:34 +0300 Subject: [PATCH 02/89] cargo lock Signed-off-by: Andrei Sandu --- Cargo.lock | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 4b0028a64513..2d75973de493 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11617,9 +11617,11 @@ name = "polkadot-approval-distribution" version = "1.0.0" dependencies = [ "assert_matches", + "bitvec", "env_logger 0.9.3", "futures", "futures-timer", + "itertools 0.10.5", "log", "polkadot-node-jaeger", "polkadot-node-metrics", @@ -11926,10 +11928,13 @@ dependencies = [ "async-trait", "bitvec", "derive_more", + "env_logger 0.9.3", "futures", "futures-timer", + "itertools 0.10.5", "kvdb", "kvdb-memorydb", + "log", "lru 0.11.0", "merlin 2.0.1", "parity-scale-codec", @@ -12398,6 +12403,7 @@ dependencies = [ name = "polkadot-node-primitives" version = "1.0.0" dependencies = [ + "bitvec", "bounded-vec", "futures", "parity-scale-codec", @@ -12448,6 +12454,7 @@ name = "polkadot-node-subsystem-types" version = "1.0.0" dependencies = [ "async-trait", + "bitvec", "derive_more", "futures", "orchestra", From 341c7af51f91f981831b79629a9141b61a875e1c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 25 Aug 2023 17:53:49 +0300 Subject: [PATCH 03/89] Approve multiple candidates with a single signature The pr migrates: - https://github.com/paritytech/polkadot/pull/7554 Signed-off-by: Alexandru Gheorghe --- .../src/blockchain_rpc_client.rs | 6 +- .../approval-voting/src/approval_checking.rs | 67 +- .../approval-voting/src/approval_db/v2/mod.rs | 34 +- .../src/approval_db/v2/tests.rs | 1 + .../node/core/approval-voting/src/criteria.rs | 2 +- .../node/core/approval-voting/src/import.rs | 3 + polkadot/node/core/approval-voting/src/lib.rs | 801 ++++++++++++++---- .../approval-voting/src/persisted_entries.rs | 133 ++- .../node/core/approval-voting/src/tests.rs | 577 ++++++++++++- .../node/core/approval-voting/src/time.rs | 165 +++- .../core/dispute-coordinator/src/import.rs | 33 +- .../dispute-coordinator/src/initialized.rs | 4 +- .../node/core/dispute-coordinator/src/lib.rs | 2 +- .../core/dispute-coordinator/src/tests.rs | 11 +- .../src/disputes/prioritized_selection/mod.rs | 2 +- polkadot/node/core/pvf/src/host.rs | 3 +- polkadot/node/core/runtime-api/src/cache.rs | 33 +- polkadot/node/core/runtime-api/src/lib.rs | 7 + polkadot/node/core/runtime-api/src/tests.rs | 17 +- .../network/approval-distribution/src/lib.rs | 576 ++++++++----- .../approval-distribution/src/metrics.rs | 160 ++++ .../approval-distribution/src/tests.rs | 211 +++-- .../network/availability-recovery/src/lib.rs | 3 +- .../network/protocol/src/grid_topology.rs | 21 +- polkadot/node/network/protocol/src/lib.rs | 7 +- polkadot/node/primitives/src/approval.rs | 57 +- .../node/primitives/src/disputes/message.rs | 2 +- polkadot/node/primitives/src/disputes/mod.rs | 23 +- polkadot/node/service/src/fake_runtime_api.rs | 13 +- polkadot/node/subsystem-types/src/messages.rs | 30 +- .../subsystem-types/src/runtime_client.rs | 21 +- polkadot/primitives/src/runtime_api.rs | 12 +- polkadot/primitives/src/v5/mod.rs | 73 +- polkadot/primitives/src/vstaging/mod.rs | 20 + .../src/node/approval/approval-voting.md | 35 +- .../src/protocol-approval.md | 6 + polkadot/runtime/kusama/src/lib.rs | 13 +- polkadot/runtime/parachains/src/builder.rs | 2 +- .../runtime/parachains/src/configuration.rs | 28 +- .../src/configuration/migration/v7.rs | 1 + .../src/configuration/migration/v8.rs | 5 +- .../parachains/src/configuration/tests.rs | 1 + polkadot/runtime/parachains/src/disputes.rs | 26 +- .../src/runtime_api_impl/vstaging.rs | 7 + polkadot/runtime/polkadot/src/lib.rs | 13 +- polkadot/runtime/rococo/src/lib.rs | 12 +- polkadot/runtime/westend/src/lib.rs | 13 +- polkadot/scripts/ci/gitlab/pipeline/build.yml | 2 +- .../scripts/ci/gitlab/pipeline/zombienet.yml | 30 + .../functional/0001-parachains-pvf.zndsl | 2 + .../functional/0002-parachains-disputes.toml | 4 + .../0006-approval-voting-coalescing.toml | 195 +++++ .../0006-approval-voting-coalescing.zndsl | 51 ++ 53 files changed, 2941 insertions(+), 635 deletions(-) create mode 100644 polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml create mode 100644 polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl diff --git a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs index fc4d803002cb..0ff3de0b296d 100644 --- a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs +++ b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs @@ -23,7 +23,7 @@ use polkadot_core_primitives::{Block, BlockNumber, Hash, Header}; use polkadot_overseer::RuntimeApiSubsystemClient; use polkadot_primitives::{ slashing, - vstaging::{AsyncBackingParams, BackingState}, + vstaging::{ApprovalVotingParams, AsyncBackingParams, BackingState}, }; use sc_authority_discovery::{AuthorityDiscovery, Error as AuthorityDiscoveryError}; use sp_api::{ApiError, RuntimeApiInfo}; @@ -349,6 +349,10 @@ impl RuntimeApiSubsystemClient for BlockChainRpcClient { ) -> Result, ApiError> { Ok(self.rpc_client.parachain_host_staging_para_backing_state(at, para_id).await?) } + /// Approval voting configuration parameters + async fn approval_voting_params(&self, _at: Hash) -> Result { + Ok(ApprovalVotingParams { max_approval_coalesce_count: 1 }) + } } #[async_trait::async_trait] diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index 5d24ff164193..2f0ea4160ae1 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -64,7 +64,7 @@ pub enum RequiredTranches { } /// The result of a check. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum Check { /// The candidate is unapproved. Unapproved, @@ -372,19 +372,22 @@ pub fn tranches_to_approve( block_tick: Tick, no_show_duration: Tick, needed_approvals: usize, -) -> RequiredTranches { +) -> (RequiredTranches, usize) { let tick_now = tranche_now as Tick + block_tick; let n_validators = approval_entry.n_validators(); - let initial_state = State { - assignments: 0, - depth: 0, - covered: 0, - covering: needed_approvals, - uncovered: 0, - next_no_show: None, - last_assignment_tick: None, - }; + let initial_state = ( + State { + assignments: 0, + depth: 0, + covered: 0, + covering: needed_approvals, + uncovered: 0, + next_no_show: None, + last_assignment_tick: None, + }, + 0usize, + ); // The `ApprovalEntry` doesn't have any data for empty tranches. We still want to iterate over // these empty tranches, so we create an iterator to fill the gaps. @@ -395,7 +398,7 @@ pub fn tranches_to_approve( tranches_with_gaps_filled .scan(Some(initial_state), |state, (tranche, assignments)| { // The `Option` here is used for early exit. - let s = state.take()?; + let (s, prev_no_shows) = state.take()?; let clock_drift = s.clock_drift(no_show_duration); let drifted_tick_now = tick_now.saturating_sub(clock_drift); @@ -444,11 +447,11 @@ pub fn tranches_to_approve( RequiredTranches::Pending { .. } => { // Pending results are only interesting when they are the last result of the iterator // i.e. we never achieve a satisfactory level of assignment. - Some(s) + Some((s, no_shows + prev_no_shows)) } }; - Some(output) + Some((output, no_shows + prev_no_shows)) }) .last() .expect("the underlying iterator is infinite, starts at 0, and never exits early before tranche 1; qed") @@ -675,7 +678,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -715,7 +719,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 2, next_no_show: Some(block_tick + no_show_duration), @@ -759,7 +764,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 11, next_no_show: None, @@ -807,7 +813,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -826,7 +833,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -879,7 +887,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -898,7 +907,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -917,7 +927,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -970,7 +981,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -992,7 +1004,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -1013,7 +1026,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Exact { needed: 3, tolerated_missing: 2, @@ -1068,7 +1082,8 @@ mod tests { block_tick, no_show_duration, needed_approvals, - ), + ) + .0, RequiredTranches::Pending { considered: 10, next_no_show: None, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs index 66df6ee8f653..88650a539130 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -17,12 +17,15 @@ //! Version 2 of the DB schema. use parity_scale_codec::{Decode, Encode}; -use polkadot_node_primitives::approval::{v1::DelayTranche, v2::AssignmentCertV2}; +use polkadot_node_primitives::approval::{ + v1::DelayTranche, + v2::{AssignmentCertV2, CandidateBitfield}, +}; use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; use polkadot_node_subsystem_util::database::{DBTransaction, Database}; use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, - ValidatorIndex, ValidatorSignature, + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, + SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; @@ -197,6 +200,16 @@ pub struct TrancheEntry { pub assignments: Vec<(ValidatorIndex, Tick)>, } +/// Metadata about our approval signature +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct OurApproval { + /// The signature for the candidates hashes pointed by indices. + pub signature: ValidatorSignature, + /// The indices of the candidates signed in this approval, an empty value means only + /// the candidate referred by this approval entry was signed. + pub signed_candidates_indices: Option, +} + /// Metadata regarding approval of a particular candidate within the context of some /// particular block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] @@ -204,7 +217,7 @@ pub struct ApprovalEntry { pub tranches: Vec, pub backing_group: GroupIndex, pub our_assignment: Option, - pub our_approval_sig: Option, + pub our_approval_sig: Option, // `n_validators` bits. pub assigned_validators: Bitfield, pub approved: bool, @@ -241,12 +254,25 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + pub candidates_pending_signature: BTreeMap, // Assignments we already distributed. A 1 bit means the candidate index for which // we already have sent out an assignment. We need this to avoid distributing // multiple core assignments more than once. pub distributed_assignments: Bitfield, } +#[derive(Encode, Decode, Debug, Clone, PartialEq)] + +/// Context needed for creating an approval signature for a given candidate. +pub struct CandidateSigningContext { + /// The candidate hash, to be included in the signature. + pub candidate_hash: CandidateHash, + /// The latest tick we have to create and send the approval. + pub send_no_later_than_tick: Tick, +} + impl From for Tick { fn from(tick: crate::Tick) -> Tick { Tick(tick) diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs index 50a5a924ca8d..31b66d88577f 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -56,6 +56,7 @@ fn make_block_entry( approved_bitfield: make_bitvec(candidates.len()), candidates, children: Vec::new(), + candidates_pending_signature: Default::default(), distributed_assignments: Default::default(), } } diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 1f751e2bf140..db74302f0225 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -278,7 +278,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) } fn check_assignment_cert( diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index 4345380ed2f9..019a74a51a74 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -513,6 +513,7 @@ pub(crate) async fn handle_new_head( .collect(), approved_bitfield, children: Vec::new(), + candidates_pending_signature: Default::default(), distributed_assignments: Default::default(), }; @@ -639,6 +640,7 @@ pub(crate) mod tests { clock: Box::new(MockClock::default()), assignment_criteria: Box::new(MockAssignmentCriteria), spans: HashMap::new(), + approval_voting_params_cache: None, } } @@ -1267,6 +1269,7 @@ pub(crate) mod tests { candidates: Vec::new(), approved_bitfield: Default::default(), children: Vec::new(), + candidates_pending_signature: Default::default(), distributed_assignments: Default::default(), } .into(), diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index beaca43ee528..69c5603624d3 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -21,14 +21,16 @@ //! of others. It uses this information to determine when candidates and blocks have //! been sufficiently approved to finalize. +use itertools::Itertools; use jaeger::{hash_to_trace_identifier, PerLeafSpan}; +use lru::LruCache; use polkadot_node_jaeger as jaeger; use polkadot_node_primitives::{ approval::{ - v1::{BlockApprovalMeta, DelayTranche, IndirectSignedApprovalVote}, + v1::{BlockApprovalMeta, DelayTranche}, v2::{ AssignmentCertKindV2, BitfieldError, CandidateBitfield, CoreBitfield, - IndirectAssignmentCertV2, + IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2, }, }, ValidationResult, DISPUTE_WINDOW, @@ -53,9 +55,10 @@ use polkadot_node_subsystem_util::{ TimeoutExt, }; use polkadot_primitives::{ - ApprovalVote, BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, DisputeStatement, - GroupIndex, Hash, PvfExecTimeoutKind, SessionIndex, SessionInfo, ValidDisputeStatementKind, - ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, + vstaging::{ApprovalVoteMultipleCandidates, ApprovalVotingParams}, + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, DisputeStatement, GroupIndex, + Hash, PvfExecTimeoutKind, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, + ValidatorIndex, ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; use sp_application_crypto::Pair; @@ -67,9 +70,11 @@ use futures::{ future::{BoxFuture, RemoteHandle}, prelude::*, stream::FuturesUnordered, + StreamExt, }; use std::{ + cmp::min, collections::{ btree_map::Entry as BTMEntry, hash_map::Entry as HMEntry, BTreeMap, HashMap, HashSet, }, @@ -82,7 +87,7 @@ use approval_checking::RequiredTranches; use bitvec::{order::Lsb0, vec::BitVec}; use criteria::{AssignmentCriteria, RealAssignmentCriteria}; use persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}; -use time::{slot_number_to_tick, Clock, ClockExt, SystemClock, Tick}; +use time::{slot_number_to_tick, Clock, ClockExt, DelayedApprovalTimer, SystemClock, Tick}; mod approval_checking; pub mod approval_db; @@ -94,9 +99,11 @@ mod persisted_entries; mod time; use crate::{ + approval_checking::Check, approval_db::v2::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, criteria::InvalidAssignmentReason, + persisted_entries::OurApproval, }; #[cfg(test)] @@ -117,6 +124,16 @@ const TICK_TOO_FAR_IN_FUTURE: Tick = 20; // 10 seconds. const APPROVAL_DELAY: Tick = 2; pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; +// The max number of ticks we delay sending the approval after we are ready to issue the approval +const MAX_APPROVAL_COALESCE_WAIT_TICKS: Tick = 12; + +// The maximum approval params we cache locally in the subsytem, so that we don't have +// to do the back and forth to the runtime subsystem api. +const APPROVAL_PARAMS_CACHE_SIZE: NonZeroUsize = match NonZeroUsize::new(128) { + Some(cap) => cap, + None => panic!("Approval params cache size must be non-zero."), +}; + /// Configuration for the approval voting subsystem #[derive(Debug, Clone)] pub struct Config { @@ -152,6 +169,9 @@ pub struct ApprovalVotingSubsystem { db: Arc, mode: Mode, metrics: Metrics, + // Store approval-voting params, so that we don't to always go to RuntimeAPI + // to ask this information which requires a task context switch. + approval_voting_params_cache: Option>, } #[derive(Clone)] @@ -160,7 +180,13 @@ struct MetricsInner { assignments_produced: prometheus::Histogram, approvals_produced_total: prometheus::CounterVec, no_shows_total: prometheus::Counter, + // The difference from `no_shows_total` is that this counts all observed no-shows at any + // moment in time. While `no_shows_total` catches that the no-shows at the moment the candidate + // is approved, approvals might arrive late and `no_shows_total` wouldn't catch that number. + observed_no_shows: prometheus::Counter, + approved_by_one_third: prometheus::Counter, wakeups_triggered_total: prometheus::Counter, + coalesced_approvals_buckets: prometheus::Histogram, candidate_approval_time_ticks: prometheus::Histogram, block_approval_time_ticks: prometheus::Histogram, time_db_transaction: prometheus::Histogram, @@ -186,6 +212,16 @@ impl Metrics { } } + fn on_approval_coalesce(&self, num_coalesced: u32) { + if let Some(metrics) = &self.0 { + // Count how many candidates we covered with this coalesced approvals, + // so that the heat-map really gives a good understanding of the scales. + for _ in 0..num_coalesced { + metrics.coalesced_approvals_buckets.observe(num_coalesced as f64) + } + } + } + fn on_approval_stale(&self) { if let Some(metrics) = &self.0 { metrics.approvals_produced_total.with_label_values(&["stale"]).inc() @@ -222,6 +258,18 @@ impl Metrics { } } + fn on_observed_no_shows(&self, n: usize) { + if let Some(metrics) = &self.0 { + metrics.observed_no_shows.inc_by(n as u64); + } + } + + fn on_approved_by_one_third(&self) { + if let Some(metrics) = &self.0 { + metrics.approved_by_one_third.inc(); + } + } + fn on_wakeup(&self) { if let Some(metrics) = &self.0 { metrics.wakeups_triggered_total.inc(); @@ -299,6 +347,13 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + observed_no_shows: prometheus::register( + prometheus::Counter::new( + "polkadot_parachain_approvals_observed_no_shows_total", + "Number of observed no shows at any moment in time", + )?, + registry, + )?, wakeups_triggered_total: prometheus::register( prometheus::Counter::new( "polkadot_parachain_approvals_wakeups_total", @@ -315,6 +370,22 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + coalesced_approvals_buckets: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_approvals_coalesced_approvals_buckets", + "Number of coalesced approvals.", + ).buckets(vec![1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5]), + )?, + registry, + )?, + approved_by_one_third: prometheus::register( + prometheus::Counter::new( + "polkadot_parachain_approved_by_one_third", + "Number of candidates where more than one third had to vote ", + )?, + registry, + )?, block_approval_time_ticks: prometheus::register( prometheus::Histogram::with_opts( prometheus::HistogramOpts::new( @@ -370,6 +441,24 @@ impl ApprovalVotingSubsystem { keystore: Arc, sync_oracle: Box, metrics: Metrics, + ) -> Self { + Self::with_config_and_cache( + config, + db, + keystore, + sync_oracle, + metrics, + Some(LruCache::new(APPROVAL_PARAMS_CACHE_SIZE)), + ) + } + + pub fn with_config_and_cache( + config: Config, + db: Arc, + keystore: Arc, + sync_oracle: Box, + metrics: Metrics, + approval_voting_params_cache: Option>, ) -> Self { ApprovalVotingSubsystem { keystore, @@ -378,6 +467,7 @@ impl ApprovalVotingSubsystem { db_config: DatabaseConfig { col_approval_data: config.col_approval_data }, mode: Mode::Syncing(sync_oracle), metrics, + approval_voting_params_cache, } } @@ -561,6 +651,7 @@ struct ApprovalStatus { required_tranches: RequiredTranches, tranche_now: DelayTranche, block_tick: Tick, + last_no_shows: usize, } #[derive(Copy, Clone)] @@ -682,6 +773,9 @@ struct State { clock: Box, assignment_criteria: Box, spans: HashMap, + // Store approval-voting params, so that we don't to always go to RuntimeAPI + // to ask this information which requires a task context switch. + approval_voting_params_cache: Option>, } #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] @@ -720,7 +814,7 @@ impl State { ); if let Some(approval_entry) = candidate_entry.approval_entry(&block_hash) { - let required_tranches = approval_checking::tranches_to_approve( + let (required_tranches, last_no_shows) = approval_checking::tranches_to_approve( approval_entry, candidate_entry.approvals(), tranche_now, @@ -729,13 +823,56 @@ impl State { session_info.needed_approvals as _, ); - let status = ApprovalStatus { required_tranches, block_tick, tranche_now }; + let status = + ApprovalStatus { required_tranches, block_tick, tranche_now, last_no_shows }; Some((approval_entry, status)) } else { None } } + + // Returns the approval voting from the RuntimeApi. + // To avoid crossing the subsystem boundary every-time we are caching locally the values. + #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] + async fn get_approval_voting_params_or_default( + &mut self, + _ctx: &mut Context, + block_hash: Hash, + ) -> ApprovalVotingParams { + if let Some(params) = self + .approval_voting_params_cache + .as_mut() + .and_then(|cache| cache.get(&block_hash)) + { + *params + } else { + // let (s_tx, s_rx) = oneshot::channel(); + + // ctx.send_message(RuntimeApiMessage::Request( + // block_hash, + // RuntimeApiRequest::ApprovalVotingParams(s_tx), + // )) + // .await; + + // match s_rx.await { + // Ok(Ok(params)) => { + // self.approval_voting_params_cache + // .as_mut() + // .map(|cache| cache.put(block_hash, params)); + // params + // }, + // _ => { + // gum::error!( + // target: LOG_TARGET, + // "Could not request approval voting params from runtime using defaults" + // ); + // TODO: Uncomment this after versi test + ApprovalVotingParams { max_approval_coalesce_count: 6 } + // }, + // } + } + } } #[derive(Debug, Clone)] @@ -784,6 +921,7 @@ where clock, assignment_criteria, spans: HashMap::new(), + approval_voting_params_cache: subsystem.approval_voting_params_cache.take(), }; // `None` on start-up. Gets initialized/updated on leaf update @@ -794,6 +932,7 @@ where let mut wakeups = Wakeups::default(); let mut currently_checking_set = CurrentlyCheckingSet::default(); let mut approvals_cache = lru::LruCache::new(APPROVAL_CACHE_SIZE); + let mut delayed_approvals_timers = DelayedApprovalTimer::default(); let mut last_finalized_height: Option = { let (tx, rx) = oneshot::channel(); @@ -871,18 +1010,50 @@ where } actions + }, + (block_hash, validator_index) = delayed_approvals_timers.select_next_some() => { + gum::debug!( + target: LOG_TARGET, + "Sign approval for multiple candidates", + ); + + let approval_params = state.get_approval_voting_params_or_default(&mut ctx, block_hash).await; + + match maybe_create_signature( + &mut overlayed_db, + &mut session_info_provider, + approval_params, + &state, &mut ctx, + block_hash, + validator_index, + &subsystem.metrics, + ).await { + Ok(Some(next_wakeup)) => { + delayed_approvals_timers.maybe_arm_timer(next_wakeup, state.clock.as_ref(), block_hash, validator_index); + }, + Ok(None) => {} + Err(err) => { + gum::error!( + target: LOG_TARGET, + ?err, + "Failed to create signature", + ); + } + } + vec![] } }; if handle_actions( &mut ctx, - &state, + &mut state, &mut overlayed_db, &mut session_info_provider, &subsystem.metrics, &mut wakeups, &mut currently_checking_set, &mut approvals_cache, + &mut delayed_approvals_timers, &mut subsystem.mode, actions, ) @@ -923,13 +1094,14 @@ where #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn handle_actions( ctx: &mut Context, - state: &State, + state: &mut State, overlayed_db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, wakeups: &mut Wakeups, currently_checking_set: &mut CurrentlyCheckingSet, approvals_cache: &mut lru::LruCache, + delayed_approvals_timers: &mut DelayedApprovalTimer, mode: &mut Mode, actions: Vec, ) -> SubsystemResult { @@ -959,6 +1131,7 @@ async fn handle_actions( session_info_provider, metrics, candidate_hash, + delayed_approvals_timers, approval_request, ) .await? @@ -1058,7 +1231,11 @@ async fn handle_actions( Action::BecomeActive => { *mode = Mode::Active; - let messages = distribution_messages_for_activation(overlayed_db, state)?; + let messages = distribution_messages_for_activation( + overlayed_db, + state, + delayed_approvals_timers, + )?; ctx.send_messages(messages.into_iter()).await; }, @@ -1084,7 +1261,7 @@ fn cores_to_candidate_indices( .iter() .position(|(core_index, _)| core_index.0 == claimed_core_index as u32) { - candidate_indices.push(candidate_index as CandidateIndex); + candidate_indices.push(candidate_index as _); } } @@ -1117,6 +1294,7 @@ fn get_assignment_core_indices( fn distribution_messages_for_activation( db: &OverlayedBackend<'_, impl Backend>, state: &State, + delayed_approvals_timers: &mut DelayedApprovalTimer, ) -> SubsystemResult> { let all_blocks: Vec = db.load_all_blocks()?; @@ -1155,7 +1333,7 @@ fn distribution_messages_for_activation( slot: block_entry.slot(), session: block_entry.session(), }); - + let mut signatures_queued = HashSet::new(); for (i, (_, candidate_hash)) in block_entry.candidates().iter().enumerate() { let _candidate_span = distribution_message_span.child("candidate").with_candidate(*candidate_hash); @@ -1183,6 +1361,15 @@ fn distribution_messages_for_activation( &candidate_hash, &block_entry, ) { + if block_entry.has_candidates_pending_signature() { + delayed_approvals_timers.maybe_arm_timer( + state.clock.tick_now(), + state.clock.as_ref(), + block_entry.block_hash(), + assignment.validator_index(), + ) + } + match cores_to_candidate_indices( &claimed_core_indices, &block_entry, @@ -1250,15 +1437,19 @@ fn distribution_messages_for_activation( continue }, } - - messages.push(ApprovalDistributionMessage::DistributeApproval( - IndirectSignedApprovalVote { - block_hash, - candidate_index: i as _, - validator: assignment.validator_index(), - signature: approval_sig, - }, - )); + let candidate_indices = approval_sig + .signed_candidates_indices + .unwrap_or((i as CandidateIndex).into()); + if signatures_queued.insert(candidate_indices.clone()) { + messages.push(ApprovalDistributionMessage::DistributeApproval( + IndirectSignedApprovalVoteV2 { + block_hash, + candidate_indices, + validator: assignment.validator_index(), + signature: approval_sig.signature, + }, + )) + }; } else { gum::warn!( target: LOG_TARGET, @@ -1464,7 +1655,7 @@ async fn get_approval_signatures_for_candidate( ctx: &mut Context, db: &OverlayedBackend<'_, impl Backend>, candidate_hash: CandidateHash, - tx: oneshot::Sender>, + tx: oneshot::Sender, ValidatorSignature)>>, ) -> SubsystemResult<()> { let send_votes = |votes| { if let Err(_) = tx.send(votes) { @@ -1490,6 +1681,11 @@ async fn get_approval_signatures_for_candidate( let relay_hashes = entry.block_assignments.keys(); let mut candidate_indices = HashSet::new(); + let mut candidate_indices_to_candidate_hashes: HashMap< + Hash, + HashMap, + > = HashMap::new(); + // Retrieve `CoreIndices`/`CandidateIndices` as required by approval-distribution: for hash in relay_hashes { let entry = match db.load_block_entry(hash)? { @@ -1507,8 +1703,11 @@ async fn get_approval_signatures_for_candidate( for (candidate_index, (_core_index, c_hash)) in entry.candidates().iter().enumerate() { if c_hash == &candidate_hash { candidate_indices.insert((*hash, candidate_index as u32)); - break } + candidate_indices_to_candidate_hashes + .entry(*hash) + .or_default() + .insert(candidate_index as _, *c_hash); } } @@ -1533,7 +1732,24 @@ async fn get_approval_signatures_for_candidate( target: LOG_TARGET, "Request for approval signatures got cancelled by `approval-distribution`." ), - Some(Ok(votes)) => send_votes(votes), + Some(Ok(votes)) => { + let votes = votes + .into_iter() + .map(|(validator_index, (hash, signed_candidates_indices, signature))| { + let candidates_hashes = + candidate_indices_to_candidate_hashes.get(&hash).expect("Can't fail because it is the same hash we sent to approval-distribution; qed"); + let signed_candidates_hashes: Vec = + signed_candidates_indices + .into_iter() + .map(|candidate_index| { + *(candidates_hashes.get(&candidate_index).expect("Can't fail because we already checked the signature was valid, so we should be able to find the hash; qed")) + }) + .collect(); + (validator_index, (signed_candidates_hashes, signature)) + }) + .collect(); + send_votes(votes) + }, } }; @@ -2167,7 +2383,7 @@ async fn check_and_import_approval( db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, - approval: IndirectSignedApprovalVote, + approval: IndirectSignedApprovalVoteV2, with_response: impl FnOnce(ApprovalCheckResult) -> T, ) -> SubsystemResult<(Vec, T)> where @@ -2179,13 +2395,12 @@ where return Ok((Vec::new(), t)) }}; } - let mut span = state .spans .get(&approval.block_hash) .map(|span| span.child("check-and-import-approval")) .unwrap_or_else(|| jaeger::Span::new(approval.block_hash, "check-and-import-approval")) - .with_uint_tag("candidate-index", approval.candidate_index as u64) + .with_string_fmt_debug_tag("candidate-index", approval.candidate_indices.clone()) .with_relay_parent(approval.block_hash) .with_stage(jaeger::Stage::ApprovalChecking); @@ -2198,105 +2413,157 @@ where }, }; - let session_info = match get_session_info( - session_info_provider, - sender, - approval.block_hash, - block_entry.session(), - ) - .await - { - Some(s) => s, - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownSessionIndex( - block_entry.session() - ),)) - }, - }; + let approved_candidates_info: Result, ApprovalCheckError> = + approval + .candidate_indices + .iter_ones() + .map(|candidate_index| { + block_entry + .candidate(candidate_index) + .ok_or(ApprovalCheckError::InvalidCandidateIndex(candidate_index as _)) + .map(|candidate| (candidate_index as _, candidate.1)) + }) + .collect(); - let approved_candidate_hash = match block_entry.candidate(approval.candidate_index as usize) { - Some((_, h)) => *h, - None => respond_early!(ApprovalCheckResult::Bad( - ApprovalCheckError::InvalidCandidateIndex(approval.candidate_index), - )), + let approved_candidates_info = match approved_candidates_info { + Ok(approved_candidates_info) => approved_candidates_info, + Err(err) => { + respond_early!(ApprovalCheckResult::Bad(err)) + }, }; - span.add_string_tag("candidate-hash", format!("{:?}", approved_candidate_hash)); + span.add_string_tag("candidate-hashes", format!("{:?}", approved_candidates_info)); span.add_string_tag( - "traceID", - format!("{:?}", hash_to_trace_identifier(approved_candidate_hash.0)), + "traceIDs", + format!( + "{:?}", + approved_candidates_info + .iter() + .map(|(_, approved_candidate_hash)| hash_to_trace_identifier( + approved_candidate_hash.0 + )) + .collect_vec() + ), ); - let pubkey = match session_info.validators.get(approval.validator) { - Some(k) => k, - None => respond_early!(ApprovalCheckResult::Bad( - ApprovalCheckError::InvalidValidatorIndex(approval.validator), - )), - }; + { + let session_info = match get_session_info( + session_info_provider, + sender, + approval.block_hash, + block_entry.session(), + ) + .await + { + Some(s) => s, + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownSessionIndex( + block_entry.session() + ),)) + }, + }; - // Signature check: - match DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking).check_signature( - &pubkey, - approved_candidate_hash, - block_entry.session(), - &approval.signature, - ) { - Err(_) => respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidSignature( - approval.validator - ),)), - Ok(()) => {}, - }; + let pubkey = match session_info.validators.get(approval.validator) { + Some(k) => k, + None => respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::InvalidValidatorIndex(approval.validator), + )), + }; - let candidate_entry = match db.load_candidate_entry(&approved_candidate_hash)? { - Some(c) => c, - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidCandidate( - approval.candidate_index, - approved_candidate_hash - ),)) - }, - }; + gum::trace!( + target: LOG_TARGET, + "Received approval for num_candidates {:}", + approval.candidate_indices.count_ones() + ); - // Don't accept approvals until assignment. - match candidate_entry.approval_entry(&approval.block_hash) { - None => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::Internal( - approval.block_hash, - approved_candidate_hash - ),)) - }, - Some(e) if !e.is_assigned(approval.validator) => { - respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::NoAssignment( - approval.validator - ),)) - }, - _ => {}, + let candidate_hashes: Vec = + approved_candidates_info.iter().map(|candidate| candidate.1).collect(); + // Signature check: + match DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone()), + ) + .check_signature( + &pubkey, + *candidate_hashes.first().expect("Checked above this is not empty; qed"), + block_entry.session(), + &approval.signature, + ) { + Err(_) => { + gum::error!( + target: LOG_TARGET, + "Error while checking signature {:}", + approval.candidate_indices.count_ones() + ); + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidSignature( + approval.validator + ),)) + }, + Ok(()) => {}, + }; } - // importing the approval can be heavy as it may trigger acceptance for a series of blocks. - let t = with_response(ApprovalCheckResult::Accepted); + let mut actions = Vec::new(); + for (approval_candidate_index, approved_candidate_hash) in approved_candidates_info { + let block_entry = match db.load_block_entry(&approval.block_hash)? { + Some(b) => b, + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::UnknownBlock( + approval.block_hash + ),)) + }, + }; - gum::trace!( - target: LOG_TARGET, - validator_index = approval.validator.0, - validator = ?pubkey, - candidate_hash = ?approved_candidate_hash, - para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, - "Importing approval vote", - ); + let candidate_entry = match db.load_candidate_entry(&approved_candidate_hash)? { + Some(c) => c, + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidCandidate( + approval_candidate_index, + approved_candidate_hash + ),)) + }, + }; - let actions = advance_approval_state( - sender, - state, - db, - session_info_provider, - &metrics, - block_entry, - approved_candidate_hash, - candidate_entry, - ApprovalStateTransition::RemoteApproval(approval.validator), - ) - .await; + // Don't accept approvals until assignment. + match candidate_entry.approval_entry(&approval.block_hash) { + None => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::Internal( + approval.block_hash, + approved_candidate_hash + ),)) + }, + Some(e) if !e.is_assigned(approval.validator) => { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::NoAssignment( + approval.validator + ),)) + }, + _ => {}, + } + + gum::debug!( + target: LOG_TARGET, + validator_index = approval.validator.0, + candidate_hash = ?approved_candidate_hash, + para_id = ?candidate_entry.candidate_receipt().descriptor.para_id, + "Importing approval vote", + ); + + let new_actions = advance_approval_state( + sender, + state, + db, + session_info_provider, + &metrics, + block_entry, + approved_candidate_hash, + candidate_entry, + ApprovalStateTransition::RemoteApproval(approval.validator), + ) + .await; + actions.extend(new_actions); + } + + // importing the approval can be heavy as it may trigger acceptance for a series of blocks. + let t = with_response(ApprovalCheckResult::Accepted); Ok((actions, t)) } @@ -2304,7 +2571,7 @@ where #[derive(Debug)] enum ApprovalStateTransition { RemoteApproval(ValidatorIndex), - LocalApproval(ValidatorIndex, ValidatorSignature), + LocalApproval(ValidatorIndex), WakeupProcessed, } @@ -2312,7 +2579,7 @@ impl ApprovalStateTransition { fn validator_index(&self) -> Option { match *self { ApprovalStateTransition::RemoteApproval(v) | - ApprovalStateTransition::LocalApproval(v, _) => Some(v), + ApprovalStateTransition::LocalApproval(v) => Some(v), ApprovalStateTransition::WakeupProcessed => None, } } @@ -2320,7 +2587,7 @@ impl ApprovalStateTransition { fn is_local_approval(&self) -> bool { match *self { ApprovalStateTransition::RemoteApproval(_) => false, - ApprovalStateTransition::LocalApproval(_, _) => true, + ApprovalStateTransition::LocalApproval(_) => true, ApprovalStateTransition::WakeupProcessed => false, } } @@ -2387,7 +2654,16 @@ where // assignment tick of `now - APPROVAL_DELAY` - that is, that // all counted assignments are at least `APPROVAL_DELAY` ticks old. let is_approved = check.is_approved(tick_now.saturating_sub(APPROVAL_DELAY)); - + if status.last_no_shows != 0 { + metrics.on_observed_no_shows(status.last_no_shows); + gum::debug!( + target: LOG_TARGET, + ?candidate_hash, + ?block_hash, + last_no_shows = ?status.last_no_shows, + "Observed no_shows", + ); + } if is_approved { gum::trace!( target: LOG_TARGET, @@ -2405,6 +2681,12 @@ where if no_shows != 0 { metrics.on_no_shows(no_shows); } + if check == Check::ApprovedOneThird { + // No-shows are not counted when more than one third of validators approve a + // candidate, so count candidates where more than one third of validators had to + // approve it, this is indicative of something breaking. + metrics.on_approved_by_one_third() + } metrics.on_candidate_approved(status.tranche_now as _); @@ -2413,6 +2695,10 @@ where actions.push(Action::NoteApprovedInChainSelection(block_hash)); } + db.write_block_entry(block_entry.into()); + } else if transition.is_local_approval() { + // Local approvals always update the block_entry, so we need to flush it to + // the database. db.write_block_entry(block_entry.into()); } @@ -2441,10 +2727,6 @@ where approval_entry.mark_approved(); } - if let ApprovalStateTransition::LocalApproval(_, ref sig) = transition { - approval_entry.import_approval_sig(sig.clone()); - } - actions.extend(schedule_wakeup_action( &approval_entry, block_hash, @@ -2569,7 +2851,7 @@ async fn process_wakeup( None => return Ok(Vec::new()), }; - let tranches_to_approve = approval_checking::tranches_to_approve( + let (tranches_to_approve, _last_no_shows) = approval_checking::tranches_to_approve( &approval_entry, candidate_entry.approvals(), tranche_now, @@ -2903,11 +3185,12 @@ async fn launch_approval( #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn issue_approval( ctx: &mut Context, - state: &State, + state: &mut State, db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, metrics: &Metrics, candidate_hash: CandidateHash, + delayed_approvals_timers: &mut DelayedApprovalTimer, ApprovalVoteRequest { validator_index, block_hash }: ApprovalVoteRequest, ) -> SubsystemResult> { let mut issue_approval_span = state @@ -2921,7 +3204,7 @@ async fn issue_approval( .with_validator_index(validator_index) .with_stage(jaeger::Stage::ApprovalChecking); - let block_entry = match db.load_block_entry(&block_hash)? { + let mut block_entry = match db.load_block_entry(&block_hash)? { Some(b) => b, None => { // not a cause for alarm - just lost a race with pruning, most likely. @@ -2947,21 +3230,6 @@ async fn issue_approval( }; issue_approval_span.add_int_tag("candidate_index", candidate_index as i64); - let session_info = match get_session_info( - session_info_provider, - ctx.sender(), - block_entry.parent_hash(), - block_entry.session(), - ) - .await - { - Some(s) => s, - None => { - metrics.on_approval_error(); - return Ok(Vec::new()) - }, - }; - let candidate_hash = match block_entry.candidate(candidate_index as usize) { Some((_, h)) => *h, None => { @@ -2992,10 +3260,150 @@ async fn issue_approval( }, }; + let session_info = match get_session_info( + session_info_provider, + ctx.sender(), + block_entry.parent_hash(), + block_entry.session(), + ) + .await + { + Some(s) => s, + None => return Ok(Vec::new()), + }; + + if block_entry + .defer_candidate_signature( + candidate_index as _, + candidate_hash, + compute_delayed_approval_sending_tick(state, &block_entry, session_info), + ) + .is_some() + { + gum::error!( + target: LOG_TARGET, + ?candidate_hash, + ?block_hash, + validator_index = validator_index.0, + "Possible bug, we shouldn't have to defer a candidate more than once", + ); + } + + gum::info!( + target: LOG_TARGET, + ?candidate_hash, + ?block_hash, + validator_index = validator_index.0, + "Ready to issue approval vote", + ); + + let approval_params = state.get_approval_voting_params_or_default(ctx, block_hash).await; + + let actions = advance_approval_state( + ctx.sender(), + state, + db, + session_info_provider, + metrics, + block_entry, + candidate_hash, + candidate_entry, + ApprovalStateTransition::LocalApproval(validator_index as _), + ) + .await; + + if let Some(next_wakeup) = maybe_create_signature( + db, + session_info_provider, + approval_params, + state, + ctx, + block_hash, + validator_index, + metrics, + ) + .await? + { + delayed_approvals_timers.maybe_arm_timer( + next_wakeup, + state.clock.as_ref(), + block_hash, + validator_index, + ); + } + Ok(actions) +} + +// Create signature for the approved candidates pending signatures +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] +async fn maybe_create_signature( + db: &mut OverlayedBackend<'_, impl Backend>, + session_info_provider: &mut RuntimeInfo, + approval_params: ApprovalVotingParams, + state: &State, + ctx: &mut Context, + block_hash: Hash, + validator_index: ValidatorIndex, + metrics: &Metrics, +) -> SubsystemResult> { + let mut block_entry = match db.load_block_entry(&block_hash)? { + Some(b) => b, + None => { + // not a cause for alarm - just lost a race with pruning, most likely. + metrics.on_approval_stale(); + gum::debug!( + target: LOG_TARGET, + "Could not find block that needs signature {:}", block_hash + ); + return Ok(None) + }, + }; + + gum::trace!( + target: LOG_TARGET, + "Candidates pending signatures {:}", block_entry.num_candidates_pending_signature() + ); + + let oldest_candidate_to_sign = match block_entry.longest_waiting_candidate_signature() { + Some(candidate) => candidate, + // No cached candidates, nothing to do here, this just means the timer fired, + // but the signatures were already sent because we gathered more than + // max_approval_coalesce_count. + None => return Ok(None), + }; + + let tick_now = state.clock.tick_now(); + + if oldest_candidate_to_sign.send_no_later_than_tick > tick_now && + (block_entry.num_candidates_pending_signature() as u32) < + approval_params.max_approval_coalesce_count + { + return Ok(Some(oldest_candidate_to_sign.send_no_later_than_tick)) + } + + let session_info = match get_session_info( + session_info_provider, + ctx.sender(), + block_entry.parent_hash(), + block_entry.session(), + ) + .await + { + Some(s) => s, + None => { + metrics.on_approval_error(); + gum::error!( + target: LOG_TARGET, + "Could not retrieve the session" + ); + return Ok(None) + }, + }; + let validator_pubkey = match session_info.validators.get(validator_index) { Some(p) => p, None => { - gum::warn!( + gum::error!( target: LOG_TARGET, "Validator index {} out of bounds in session {}", validator_index.0, @@ -3003,72 +3411,93 @@ async fn issue_approval( ); metrics.on_approval_error(); - return Ok(Vec::new()) + return Ok(None) }, }; - let session = block_entry.session(); - let sig = match sign_approval(&state.keystore, &validator_pubkey, candidate_hash, session) { + let candidate_hashes = block_entry.candidate_hashes_pending_signature(); + + let signature = match sign_approval( + &state.keystore, + &validator_pubkey, + candidate_hashes.clone(), + block_entry.session(), + ) { Some(sig) => sig, None => { - gum::warn!( + gum::error!( target: LOG_TARGET, validator_index = ?validator_index, - session, + session = ?block_entry.session(), "Could not issue approval signature. Assignment key present but not validator key?", ); metrics.on_approval_error(); - return Ok(Vec::new()) + return Ok(None) }, }; - gum::trace!( - target: LOG_TARGET, - ?candidate_hash, - ?block_hash, - validator_index = validator_index.0, - "Issuing approval vote", - ); + let candidate_entries = candidate_hashes + .iter() + .map(|candidate_hash| db.load_candidate_entry(candidate_hash)) + .collect::>>>()?; - let actions = advance_approval_state( - ctx.sender(), - state, - db, - session_info_provider, - metrics, - block_entry, - candidate_hash, - candidate_entry, - ApprovalStateTransition::LocalApproval(validator_index as _, sig.clone()), - ) - .await; + let candidate_indices = block_entry.candidate_indices_pending_signature(); + + for candidate_entry in candidate_entries { + let mut candidate_entry = candidate_entry + .expect("Candidate was scheduled to be signed entry in db should exist; qed"); + let approval_entry = candidate_entry + .approval_entry_mut(&block_entry.block_hash()) + .expect("Candidate was scheduled to be signed entry in db should exist; qed"); + approval_entry.import_approval_sig(OurApproval { + signature: signature.clone(), + signed_candidates_indices: Some( + candidate_indices + .clone() + .try_into() + .expect("Fails only of array empty, it can't be, qed"), + ), + }); + db.write_candidate_entry(candidate_entry); + } + + metrics.on_approval_coalesce(candidate_indices.len() as u32); metrics.on_approval_produced(); - // dispatch to approval distribution. ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeApproval( - IndirectSignedApprovalVote { - block_hash, - candidate_index: candidate_index as _, + IndirectSignedApprovalVoteV2 { + block_hash: block_entry.block_hash(), + candidate_indices: candidate_indices + .try_into() + .expect("Fails only of array empty, it can't be, qed"), validator: validator_index, - signature: sig, + signature, }, )); - Ok(actions) + gum::trace!( + target: LOG_TARGET, + ?block_hash, + signed_candidates = ?block_entry.num_candidates_pending_signature(), + "Issue approval votes", + ); + block_entry.issued_approval(); + db.write_block_entry(block_entry.into()); + Ok(None) } // Sign an approval vote. Fails if the key isn't present in the store. fn sign_approval( keystore: &LocalKeystore, public: &ValidatorId, - candidate_hash: CandidateHash, + candidate_hashes: Vec, session_index: SessionIndex, ) -> Option { let key = keystore.key_pair::(public).ok().flatten()?; - let payload = ApprovalVote(candidate_hash).signing_payload(session_index); + let payload = ApprovalVoteMultipleCandidates(&candidate_hashes).signing_payload(session_index); Some(key.sign(&payload[..])) } @@ -3098,3 +3527,27 @@ fn issue_local_invalid_statement( false, )); } + +// Computes what is the latest tick we can send an approval +fn compute_delayed_approval_sending_tick( + state: &State, + block_entry: &BlockEntry, + session_info: &SessionInfo, +) -> Tick { + let current_block_tick = slot_number_to_tick(state.slot_duration_millis, block_entry.slot()); + + let no_show_duration_ticks = slot_number_to_tick( + state.slot_duration_millis, + Slot::from(u64::from(session_info.no_show_slots)), + ); + let tick_now = state.clock.tick_now(); + + min( + tick_now + MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick, + // We don't want to accidentally cause, no-shows so if we are past + // the 2 thirds of the no show time, force the sending of the + // approval immediately. + // TODO: TBD the right value here, so that we don't accidentally create no-shows. + current_block_tick + (no_show_duration_ticks * 2) / 3, + ) +} diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 155b2f9c4e02..e441c23be122 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -25,8 +25,8 @@ use polkadot_node_primitives::approval::{ v2::{AssignmentCertV2, CandidateBitfield}, }; use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex, - ValidatorIndex, ValidatorSignature, + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, + SessionIndex, ValidatorIndex, ValidatorSignature, }; use sp_consensus_slots::Slot; @@ -76,6 +76,39 @@ impl From for crate::approval_db::v2::TrancheEntry { } } +impl From for OurApproval { + fn from(approval: crate::approval_db::v2::OurApproval) -> Self { + Self { + signature: approval.signature, + signed_candidates_indices: approval.signed_candidates_indices, + } + } +} +impl From for crate::approval_db::v2::OurApproval { + fn from(approval: OurApproval) -> Self { + Self { + signature: approval.signature, + signed_candidates_indices: approval.signed_candidates_indices, + } + } +} + +impl From for OurApproval { + fn from(value: ValidatorSignature) -> Self { + Self { signature: value, signed_candidates_indices: Default::default() } + } +} + +/// Metadata about our approval signature +#[derive(Debug, Clone, PartialEq)] +pub struct OurApproval { + /// The signature for the candidates hashes pointed by indices. + pub signature: ValidatorSignature, + /// The indices of the candidates signed in this approval, an empty value means only + /// the candidate referred by this approval entry was signed. + pub signed_candidates_indices: Option, +} + /// Metadata regarding approval of a particular candidate within the context of some /// particular block. #[derive(Debug, Clone, PartialEq)] @@ -83,7 +116,7 @@ pub struct ApprovalEntry { tranches: Vec, backing_group: GroupIndex, our_assignment: Option, - our_approval_sig: Option, + our_approval_sig: Option, // `n_validators` bits. assigned_validators: Bitfield, approved: bool, @@ -95,7 +128,7 @@ impl ApprovalEntry { tranches: Vec, backing_group: GroupIndex, our_assignment: Option, - our_approval_sig: Option, + our_approval_sig: Option, // `n_validators` bits. assigned_validators: Bitfield, approved: bool, @@ -137,7 +170,7 @@ impl ApprovalEntry { } /// Import our local approval vote signature for this candidate. - pub fn import_approval_sig(&mut self, approval_sig: ValidatorSignature) { + pub fn import_approval_sig(&mut self, approval_sig: OurApproval) { self.our_approval_sig = Some(approval_sig); } @@ -224,7 +257,7 @@ impl ApprovalEntry { /// Get the assignment cert & approval signature. /// /// The approval signature will only be `Some` if the assignment is too. - pub fn local_statements(&self) -> (Option, Option) { + pub fn local_statements(&self) -> (Option, Option) { let approval_sig = self.our_approval_sig.clone(); if let Some(our_assignment) = self.our_assignment.as_ref().filter(|a| a.triggered()) { (Some(our_assignment.clone()), approval_sig) @@ -353,12 +386,21 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + candidates_pending_signature: BTreeMap, // A list of assignments for which wea already distributed the assignment. // We use this to ensure we don't distribute multiple core assignments twice as we track // individual wakeups for each core. distributed_assignments: Bitfield, } +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateSigningContext { + pub candidate_hash: CandidateHash, + pub send_no_later_than_tick: Tick, +} + impl BlockEntry { /// Mark a candidate as fully approved in the bitfield. pub fn mark_approved_by_hash(&mut self, candidate_hash: &CandidateHash) { @@ -447,6 +489,54 @@ impl BlockEntry { distributed } + + /// Defer signing and issuing an approval for a candidate no later than the specified tick + pub fn defer_candidate_signature( + &mut self, + candidate_index: CandidateIndex, + candidate_hash: CandidateHash, + send_no_later_than_tick: Tick, + ) -> Option { + self.candidates_pending_signature.insert( + candidate_index, + CandidateSigningContext { candidate_hash, send_no_later_than_tick }, + ) + } + + /// Returns the number of candidates waiting for an approval to be issued. + pub fn num_candidates_pending_signature(&self) -> usize { + self.candidates_pending_signature.len() + } + + /// Return if we have candidates waiting for signature to be issued + pub fn has_candidates_pending_signature(&self) -> bool { + !self.candidates_pending_signature.is_empty() + } + + /// Candidate hashes for candidates pending signatures + pub fn candidate_hashes_pending_signature(&self) -> Vec { + self.candidates_pending_signature + .values() + .map(|unsigned_approval| unsigned_approval.candidate_hash) + .collect() + } + + /// Candidate indices for candidates pending signature + pub fn candidate_indices_pending_signature(&self) -> Vec { + self.candidates_pending_signature.keys().map(|val| *val).collect() + } + + /// Returns the candidate that has been longest in the queue. + pub fn longest_waiting_candidate_signature(&self) -> Option<&CandidateSigningContext> { + self.candidates_pending_signature + .values() + .min_by(|a, b| a.send_no_later_than_tick.cmp(&b.send_no_later_than_tick)) + } + + /// Signals the approval was issued for the candidates pending signature + pub fn issued_approval(&mut self) { + self.candidates_pending_signature.clear(); + } } impl From for BlockEntry { @@ -461,6 +551,11 @@ impl From for BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + candidates_pending_signature: entry + .candidates_pending_signature + .into_iter() + .map(|(candidate_index, signing_context)| (candidate_index, signing_context.into())) + .collect(), distributed_assignments: entry.distributed_assignments, } } @@ -479,6 +574,7 @@ impl From for BlockEntry { approved_bitfield: entry.approved_bitfield, children: entry.children, distributed_assignments: Default::default(), + candidates_pending_signature: Default::default(), } } } @@ -495,11 +591,34 @@ impl From for crate::approval_db::v2::BlockEntry { candidates: entry.candidates, approved_bitfield: entry.approved_bitfield, children: entry.children, + candidates_pending_signature: entry + .candidates_pending_signature + .into_iter() + .map(|(candidate_index, signing_context)| (candidate_index, signing_context.into())) + .collect(), distributed_assignments: entry.distributed_assignments, } } } +impl From for CandidateSigningContext { + fn from(signing_context: crate::approval_db::v2::CandidateSigningContext) -> Self { + Self { + candidate_hash: signing_context.candidate_hash, + send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), + } + } +} + +impl From for crate::approval_db::v2::CandidateSigningContext { + fn from(signing_context: CandidateSigningContext) -> Self { + Self { + candidate_hash: signing_context.candidate_hash, + send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), + } + } +} + /// Migration helpers. impl From for CandidateEntry { fn from(value: crate::approval_db::v1::CandidateEntry) -> Self { @@ -522,7 +641,7 @@ impl From for ApprovalEntry { tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), backing_group: value.backing_group, our_assignment: value.our_assignment.map(|assignment| assignment.into()), - our_approval_sig: value.our_approval_sig, + our_approval_sig: value.our_approval_sig.map(Into::into), assigned_validators: value.assignments, approved: value.approved, } diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index c2ef109ad4ca..b8ae3003efd8 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -36,8 +36,8 @@ use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_overseer::HeadSupportsParachains; use polkadot_primitives::{ - CandidateCommitments, CandidateEvent, CoreIndex, GroupIndex, Header, Id as ParaId, IndexedVec, - ValidationCode, ValidatorSignature, + ApprovalVote, CandidateCommitments, CandidateEvent, CoreIndex, GroupIndex, Header, + Id as ParaId, IndexedVec, ValidationCode, ValidatorSignature, }; use std::time::Duration; @@ -438,6 +438,15 @@ fn sign_approval( key.sign(&ApprovalVote(candidate_hash).signing_payload(session_index)).into() } +fn sign_approval_multiple_candidates( + key: Sr25519Keyring, + candidate_hashes: Vec, + session_index: SessionIndex, +) -> ValidatorSignature { + key.sign(&ApprovalVoteMultipleCandidates(&candidate_hashes).signing_payload(session_index)) + .into() +} + type VirtualOverseer = test_helpers::TestSubsystemContextHandle; #[derive(Default)] @@ -530,7 +539,7 @@ fn test_harness>( let subsystem = run( context, - ApprovalVotingSubsystem::with_config( + ApprovalVotingSubsystem::with_config_and_cache( Config { col_approval_data: test_constants::TEST_CONFIG.col_approval_data, slot_duration_millis: SLOT_DURATION_MILLIS, @@ -539,6 +548,7 @@ fn test_harness>( Arc::new(keystore), sync_oracle, Metrics::default(), + None, ), clock.clone(), assignment_criteria, @@ -633,7 +643,12 @@ async fn check_and_import_approval( overseer, FromOrchestra::Communication { msg: ApprovalVotingMessage::CheckAndImportApproval( - IndirectSignedApprovalVote { block_hash, candidate_index, validator, signature }, + IndirectSignedApprovalVoteV2 { + block_hash, + candidate_indices: candidate_index.into(), + validator, + signature, + }, tx, ), }, @@ -1989,6 +2004,91 @@ fn forkful_import_at_same_height_act_on_leaf() { }); } +#[test] +fn test_signing_a_single_candidate_is_backwards_compatible() { + let session_index = 1; + let block_hash = Hash::repeat_byte(0x01); + let candidate_descriptors = (1..10) + .into_iter() + .map(|val| make_candidate(ParaId::from(val as u32), &block_hash)) + .collect::>(); + + let candidate_hashes = candidate_descriptors + .iter() + .map(|candidate_descriptor| candidate_descriptor.hash()) + .collect_vec(); + + let first_descriptor = candidate_descriptors.first().expect("TODO"); + + let candidate_hash = first_descriptor.hash(); + + let sig_a = sign_approval(Sr25519Keyring::Alice, candidate_hash, session_index); + + let sig_b = sign_approval(Sr25519Keyring::Alice, candidate_hash, session_index); + + assert!(DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + candidate_hash, + session_index, + &sig_a, + ) + .is_ok()); + + assert!(DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + candidate_hash, + session_index, + &sig_b, + ) + .is_ok()); + + let sig_c = sign_approval_multiple_candidates( + Sr25519Keyring::Alice, + vec![candidate_hash], + session_index, + ); + + assert!(DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(vec![candidate_hash]) + ) + .check_signature(&Sr25519Keyring::Alice.public().into(), candidate_hash, session_index, &sig_c,) + .is_ok()); + + assert!(DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + candidate_hash, + session_index, + &sig_c, + ) + .is_ok()); + + assert!(DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(vec![candidate_hash]) + ) + .check_signature(&Sr25519Keyring::Alice.public().into(), candidate_hash, session_index, &sig_a,) + .is_ok()); + + let sig_all = sign_approval_multiple_candidates( + Sr25519Keyring::Alice, + candidate_hashes.clone(), + session_index, + ); + + assert!(DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone()) + ) + .check_signature( + &Sr25519Keyring::Alice.public().into(), + *candidate_hashes.first().expect("test"), + session_index, + &sig_all, + ) + .is_ok()); +} + #[test] fn import_checked_approval_updates_entries_and_schedules() { let config = HarnessConfig::default(); @@ -2701,11 +2801,29 @@ async fn handle_double_assignment_import( } ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) @@ -3440,3 +3558,454 @@ fn waits_until_approving_assignments_are_old_enough() { virtual_overseer }); } + +#[test] +fn test_approval_is_sent_on_max_approval_coalesce_count() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let assignments_cert = + garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _sync_oracle_handle } = + test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + + let candidate_commitments = CandidateCommitments::default(); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let candidate_hash1 = candidate_receipt1.hash(); + + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let slot = Slot::from(1); + let candidate_index1 = 0; + let candidate_index2 = 1; + + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Eve, + ]; + let session_info = SessionInfo { + validator_groups: IndexedVec::>::from(vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2)], + vec![ValidatorIndex(3), ValidatorIndex(4)], + ]), + ..session_info(&validators) + }; + + let candidates = Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(0)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]); + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates: candidates.clone(), + session_info: Some(session_info.clone()), + }, + ) + .build(&mut virtual_overseer) + .await; + + assert!(!clock.inner.lock().current_wakeup_is(1)); + clock.inner.lock().wakeup_all(1); + + assert!(clock.inner.lock().current_wakeup_is(slot_to_tick(slot))); + clock.inner.lock().wakeup_all(slot_to_tick(slot)); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + clock.inner.lock().wakeup_all(slot_to_tick(slot + 2)); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + let candidate_entry = store.load_candidate_entry(&candidate_hash1).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + handle_approval_on_max_coalesce_count( + &mut virtual_overseer, + vec![candidate_index1, candidate_index2], + ) + .await; + + virtual_overseer + }); +} + +async fn handle_approval_on_max_coalesce_count( + virtual_overseer: &mut VirtualOverseer, + candidate_indicies: Vec, +) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(TryInto::::try_into(candidate_indicies.clone()).unwrap(), c_indices); + } + ); + + for _ in &candidate_indicies { + recover_available_data(virtual_overseer).await; + fetch_validation_code(virtual_overseer).await; + } + + for _ in &candidate_indicies { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + } + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 2, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 2, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(vote)) => { + assert_eq!(TryInto::::try_into(candidate_indicies).unwrap(), vote.candidate_indices); + } + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); +} + +async fn handle_approval_on_max_wait_time( + virtual_overseer: &mut VirtualOverseer, + candidate_indicies: Vec, + clock: Box, +) { + const TICK_NOW_BEGIN: u64 = 1; + const MAX_COALESCE_COUNT: u32 = 3; + + clock.inner.lock().set_tick(TICK_NOW_BEGIN); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + c_indices, + )) => { + assert_eq!(TryInto::::try_into(candidate_indicies.clone()).unwrap(), c_indices); + } + ); + + for _ in &candidate_indicies { + recover_available_data(virtual_overseer).await; + fetch_validation_code(virtual_overseer).await; + } + + for _ in &candidate_indicies { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + } + + // First time we fetch the configuration when we are ready to approve the first candidate + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: MAX_COALESCE_COUNT, + })); + } + ); + + // Second time we fetch the configuration when we are ready to approve the second candidate + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: MAX_COALESCE_COUNT, + })); + } + ); + + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + // Move the clock just before we should send the approval + clock + .inner + .lock() + .set_tick(MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick + TICK_NOW_BEGIN - 1); + + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + // Move the clock tick, so we can trigger a force sending of the approvals + clock + .inner + .lock() + .set_tick(MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick + TICK_NOW_BEGIN); + + // Third time we fetch the configuration when timer expires and we are ready to sent the approval + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 3, + })); + } + ); + + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(vote)) => { + assert_eq!(TryInto::::try_into(candidate_indicies).unwrap(), vote.candidate_indices); + } + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); +} + +#[test] +fn test_approval_is_sent_on_max_approval_coalesce_wait() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let assignments_cert = + garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(1), + approval_db::v2::OurAssignment { + cert: assignments_cert.clone(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _sync_oracle_handle } = + test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + + let candidate_commitments = CandidateCommitments::default(); + + let candidate_receipt1 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(1_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let candidate_hash1 = candidate_receipt1.hash(); + + let candidate_receipt2 = { + let mut receipt = dummy_candidate_receipt(block_hash); + receipt.descriptor.para_id = ParaId::from(2_u32); + receipt.commitments_hash = candidate_commitments.hash(); + receipt + }; + + let slot = Slot::from(1); + let candidate_index1 = 0; + let candidate_index2 = 1; + + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Eve, + ]; + let session_info = SessionInfo { + validator_groups: IndexedVec::>::from(vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2)], + vec![ValidatorIndex(3), ValidatorIndex(4)], + ]), + ..session_info(&validators) + }; + + let candidates = Some(vec![ + (candidate_receipt1.clone(), CoreIndex(0), GroupIndex(0)), + (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), + ]); + ChainBuilder::new() + .add_block( + block_hash, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates: candidates.clone(), + session_info: Some(session_info.clone()), + }, + ) + .build(&mut virtual_overseer) + .await; + + assert!(!clock.inner.lock().current_wakeup_is(1)); + clock.inner.lock().wakeup_all(1); + + assert!(clock.inner.lock().current_wakeup_is(slot_to_tick(slot))); + clock.inner.lock().wakeup_all(slot_to_tick(slot)); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + clock.inner.lock().wakeup_all(slot_to_tick(slot + 2)); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + let candidate_entry = store.load_candidate_entry(&candidate_hash1).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); + + handle_approval_on_max_wait_time( + &mut virtual_overseer, + vec![candidate_index1, candidate_index2], + clock, + ) + .await; + + virtual_overseer + }); +} diff --git a/polkadot/node/core/approval-voting/src/time.rs b/polkadot/node/core/approval-voting/src/time.rs index a45866402c82..61091f3c34cd 100644 --- a/polkadot/node/core/approval-voting/src/time.rs +++ b/polkadot/node/core/approval-voting/src/time.rs @@ -16,14 +16,23 @@ //! Time utilities for approval voting. -use futures::prelude::*; +use futures::{ + future::BoxFuture, + prelude::*, + stream::{FusedStream, FuturesUnordered}, + Stream, StreamExt, +}; + use polkadot_node_primitives::approval::v1::DelayTranche; use sp_consensus_slots::Slot; use std::{ + collections::HashSet, pin::Pin, + task::Poll, time::{Duration, SystemTime}, }; +use polkadot_primitives::{Hash, ValidatorIndex}; const TICK_DURATION_MILLIS: u64 = 500; /// A base unit of time, starting from the Unix epoch, split into half-second intervals. @@ -88,3 +97,157 @@ pub(crate) fn slot_number_to_tick(slot_duration_millis: u64, slot: Slot) -> Tick let ticks_per_slot = slot_duration_millis / TICK_DURATION_MILLIS; u64::from(slot) * ticks_per_slot } + +/// A list of delayed futures that gets triggered when the waiting time has expired and it is +/// time to sign the candidate. +/// We have a timer per relay-chain block. +#[derive(Default)] +pub struct DelayedApprovalTimer { + timers: FuturesUnordered>, + blocks: HashSet, +} + +impl DelayedApprovalTimer { + /// Starts a single timer per block hash + /// + /// Guarantees that if a timer already exits for the give block hash, + /// no additional timer is started. + pub(crate) fn maybe_arm_timer( + &mut self, + wait_untill: Tick, + clock: &dyn Clock, + block_hash: Hash, + validator_index: ValidatorIndex, + ) { + if self.blocks.insert(block_hash) { + let clock_wait = clock.wait(wait_untill); + self.timers.push(Box::pin(async move { + clock_wait.await; + (block_hash, validator_index) + })); + } + } +} + +impl Stream for DelayedApprovalTimer { + type Item = (Hash, ValidatorIndex); + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let poll_result = self.timers.poll_next_unpin(cx); + match poll_result { + Poll::Ready(Some(result)) => { + self.blocks.remove(&result.0); + Poll::Ready(Some(result)) + }, + _ => poll_result, + } + } +} + +impl FusedStream for DelayedApprovalTimer { + fn is_terminated(&self) -> bool { + self.timers.is_terminated() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use futures::{executor::block_on, FutureExt, StreamExt}; + use futures_timer::Delay; + use polkadot_primitives::{Hash, ValidatorIndex}; + + use crate::time::{Clock, SystemClock}; + + use super::DelayedApprovalTimer; + + #[test] + fn test_select_empty_timer() { + block_on(async move { + let mut timer = DelayedApprovalTimer::default(); + + for _ in 1..10 { + let result = futures::select!( + _ = timer.select_next_some() => { + 0 + } + // Only this arm should fire + _ = Delay::new(Duration::from_millis(100)).fuse() => { + 1 + } + ); + + assert_eq!(result, 1); + } + }); + } + + #[test] + fn test_timer_functionality() { + block_on(async move { + let mut timer = DelayedApprovalTimer::default(); + let test_hashes = + vec![Hash::repeat_byte(0x01), Hash::repeat_byte(0x02), Hash::repeat_byte(0x03)]; + for (index, hash) in test_hashes.iter().enumerate() { + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + } + let timeout_hash = Hash::repeat_byte(0x02); + for i in 0..test_hashes.len() * 2 { + let result = futures::select!( + (hash, _) = timer.select_next_some() => { + hash + } + // Timers should fire only once, so for the rest of the iterations we should timeout through here. + _ = Delay::new(Duration::from_secs(2)).fuse() => { + timeout_hash + } + ); + assert_eq!(test_hashes.get(i).cloned().unwrap_or(timeout_hash), result); + } + + // Now check timer can be restarted if already fired + for (index, hash) in test_hashes.iter().enumerate() { + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + timer.maybe_arm_timer( + SystemClock.tick_now() + index as u64, + &SystemClock, + *hash, + ValidatorIndex::from(2), + ); + } + + for i in 0..test_hashes.len() * 2 { + let result = futures::select!( + (hash, _) = timer.select_next_some() => { + hash + } + // Timers should fire only once, so for the rest of the iterations we should timeout through here. + _ = Delay::new(Duration::from_secs(2)).fuse() => { + timeout_hash + } + ); + assert_eq!(test_hashes.get(i).cloned().unwrap_or(timeout_hash), result); + } + }); + } +} diff --git a/polkadot/node/core/dispute-coordinator/src/import.rs b/polkadot/node/core/dispute-coordinator/src/import.rs index 6222aab1cb10..d2520e6f9a64 100644 --- a/polkadot/node/core/dispute-coordinator/src/import.rs +++ b/polkadot/node/core/dispute-coordinator/src/import.rs @@ -34,7 +34,7 @@ use polkadot_node_primitives::{ use polkadot_node_subsystem::overseer; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::{ - CandidateReceipt, DisputeStatement, Hash, IndexedVec, SessionIndex, SessionInfo, + CandidateHash, CandidateReceipt, DisputeStatement, Hash, IndexedVec, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; @@ -118,7 +118,9 @@ impl OwnVoteState { let our_valid_votes = controlled_indices .iter() .filter_map(|i| votes.valid.raw().get_key_value(i)) - .map(|(index, (kind, sig))| (*index, (DisputeStatement::Valid(*kind), sig.clone()))); + .map(|(index, (kind, sig))| { + (*index, (DisputeStatement::Valid(kind.clone()), sig.clone())) + }); let our_invalid_votes = controlled_indices .iter() .filter_map(|i| votes.invalid.get_key_value(i)) @@ -297,7 +299,7 @@ impl CandidateVoteState { DisputeStatement::Valid(valid_kind) => { let fresh = votes.valid.insert_vote( val_index, - *valid_kind, + valid_kind.clone(), statement.into_validator_signature(), ); if fresh { @@ -503,7 +505,7 @@ impl ImportResult { pub fn import_approval_votes( self, env: &CandidateEnvironment, - approval_votes: HashMap, + approval_votes: HashMap, ValidatorSignature)>, now: Timestamp, ) -> Self { let Self { @@ -517,19 +519,32 @@ impl ImportResult { let (mut votes, _) = new_state.into_old_state(); - for (index, sig) in approval_votes.into_iter() { + for (index, (candidate_hashes, sig)) in approval_votes.into_iter() { debug_assert!( { let pub_key = &env.session_info().validators.get(index).expect("indices are validated by approval-voting subsystem; qed"); - let candidate_hash = votes.candidate_receipt.hash(); let session_index = env.session_index(); - DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) - .check_signature(pub_key, candidate_hash, session_index, &sig) + candidate_hashes.contains(&votes.candidate_receipt.hash()) && DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes.clone())) + .check_signature(pub_key, *candidate_hashes.first().expect("Valid votes have at least one candidate; qed"), session_index, &sig) .is_ok() }, "Signature check for imported approval votes failed! This is a serious bug. Session: {:?}, candidate hash: {:?}, validator index: {:?}", env.session_index(), votes.candidate_receipt.hash(), index ); - if votes.valid.insert_vote(index, ValidDisputeStatementKind::ApprovalChecking, sig) { + if votes.valid.insert_vote( + index, + // There is a hidden dependency here between approval-voting and this subsystem. + // We should be able to start emitting ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates only after: + // 1. Runtime have been upgraded to know about the new format. + // 2. All nodes have been upgraded to know about the new format. + // Once those two requirements have been met we should be able to increase max_approval_coalesce_count to values + // greater than 1. + if candidate_hashes.len() > 1 { + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes) + } else { + ValidDisputeStatementKind::ApprovalChecking + }, + sig, + ) { imported_valid_votes += 1; imported_approval_votes += 1; } diff --git a/polkadot/node/core/dispute-coordinator/src/initialized.rs b/polkadot/node/core/dispute-coordinator/src/initialized.rs index c1d02ef976cb..7afc120d4161 100644 --- a/polkadot/node/core/dispute-coordinator/src/initialized.rs +++ b/polkadot/node/core/dispute-coordinator/src/initialized.rs @@ -633,7 +633,7 @@ impl Initialized { }; debug_assert!( SignedDisputeStatement::new_checked( - DisputeStatement::Valid(valid_statement_kind), + DisputeStatement::Valid(valid_statement_kind.clone()), candidate_hash, session, validator_public.clone(), @@ -647,7 +647,7 @@ impl Initialized { ); let signed_dispute_statement = SignedDisputeStatement::new_unchecked_from_trusted_source( - DisputeStatement::Valid(valid_statement_kind), + DisputeStatement::Valid(valid_statement_kind.clone()), candidate_hash, session, validator_public, diff --git a/polkadot/node/core/dispute-coordinator/src/lib.rs b/polkadot/node/core/dispute-coordinator/src/lib.rs index a2c500e08e28..b18cf8aa4aa9 100644 --- a/polkadot/node/core/dispute-coordinator/src/lib.rs +++ b/polkadot/node/core/dispute-coordinator/src/lib.rs @@ -575,7 +575,7 @@ pub fn make_dispute_message( .next() .ok_or(DisputeMessageCreationError::NoOppositeVote)?; let other_vote = SignedDisputeStatement::new_checked( - DisputeStatement::Valid(*statement_kind), + DisputeStatement::Valid(statement_kind.clone()), *our_vote.candidate_hash(), our_vote.session_index(), validators diff --git a/polkadot/node/core/dispute-coordinator/src/tests.rs b/polkadot/node/core/dispute-coordinator/src/tests.rs index 75eae8200dc6..552613c566b9 100644 --- a/polkadot/node/core/dispute-coordinator/src/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/tests.rs @@ -651,7 +651,7 @@ fn make_candidate_included_event(candidate_receipt: CandidateReceipt) -> Candida pub async fn handle_approval_vote_request( ctx_handle: &mut VirtualOverseer, expected_hash: &CandidateHash, - votes_to_send: HashMap, + votes_to_send: HashMap, ValidatorSignature)>, ) { assert_matches!( ctx_handle.recv().await, @@ -858,9 +858,12 @@ fn approval_vote_import_works() { .await; gum::trace!("After sending `ImportStatements`"); - let approval_votes = [(ValidatorIndex(4), approval_vote.into_validator_signature())] - .into_iter() - .collect(); + let approval_votes = [( + ValidatorIndex(4), + (vec![candidate_receipt1.hash()], approval_vote.into_validator_signature()), + )] + .into_iter() + .collect(); handle_approval_vote_request(&mut virtual_overseer, &candidate_hash1, approval_votes) .await; diff --git a/polkadot/node/core/provisioner/src/disputes/prioritized_selection/mod.rs b/polkadot/node/core/provisioner/src/disputes/prioritized_selection/mod.rs index 096b73d271a8..cb55ce39bc89 100644 --- a/polkadot/node/core/provisioner/src/disputes/prioritized_selection/mod.rs +++ b/polkadot/node/core/provisioner/src/disputes/prioritized_selection/mod.rs @@ -221,7 +221,7 @@ where votes.valid.retain(|validator_idx, (statement_kind, _)| { is_vote_worth_to_keep( validator_idx, - DisputeStatement::Valid(*statement_kind), + DisputeStatement::Valid(statement_kind.clone()), &onchain_state, ) }); diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 9f3b7e23fd89..aadc14775025 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -186,7 +186,8 @@ impl Config { prepare_workers_hard_max_num: 1, execute_worker_program_path, execute_worker_spawn_timeout: Duration::from_secs(3), - execute_workers_max_num: 2, + // TODO: cleanup increased for versi experimenting. + execute_workers_max_num: 4, } } } diff --git a/polkadot/node/core/runtime-api/src/cache.rs b/polkadot/node/core/runtime-api/src/cache.rs index 26aaf3fb6ec8..a0b2c7c0f2e4 100644 --- a/polkadot/node/core/runtime-api/src/cache.rs +++ b/polkadot/node/core/runtime-api/src/cache.rs @@ -20,12 +20,12 @@ use lru::LruCache; use sp_consensus_babe::Epoch; use polkadot_primitives::{ - vstaging, AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + vstaging::{self, ApprovalVotingParams}, + AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; /// For consistency we have the same capacity for all caches. We use 128 as we'll only need that @@ -64,6 +64,8 @@ pub(crate) struct RequestResultCache { LruCache<(Hash, ParaId, OccupiedCoreAssumption), Option>, version: LruCache, disputes: LruCache)>>, + approval_voting_params: LruCache, + unapplied_slashes: LruCache>, key_ownership_proof: @@ -100,7 +102,7 @@ impl Default for RequestResultCache { disputes: LruCache::new(DEFAULT_CACHE_CAP), unapplied_slashes: LruCache::new(DEFAULT_CACHE_CAP), key_ownership_proof: LruCache::new(DEFAULT_CACHE_CAP), - + approval_voting_params: LruCache::new(DEFAULT_CACHE_CAP), staging_para_backing_state: LruCache::new(DEFAULT_CACHE_CAP), staging_async_backing_params: LruCache::new(DEFAULT_CACHE_CAP), } @@ -399,6 +401,21 @@ impl RequestResultCache { self.disputes.put(relay_parent, value); } + pub(crate) fn approval_voting_params( + &mut self, + relay_parent: &Hash, + ) -> Option<&ApprovalVotingParams> { + self.approval_voting_params.get(relay_parent) + } + + pub(crate) fn cache_approval_voting_params( + &mut self, + relay_parent: Hash, + value: ApprovalVotingParams, + ) { + self.approval_voting_params.put(relay_parent, value); + } + pub(crate) fn unapplied_slashes( &mut self, relay_parent: &Hash, @@ -512,7 +529,7 @@ pub(crate) enum RequestResult { vstaging::slashing::OpaqueKeyOwnershipProof, Option<()>, ), - + ApprovalVotingParams(Hash, ApprovalVotingParams), StagingParaBackingState(Hash, ParaId, Option), StagingAsyncBackingParams(Hash, vstaging::AsyncBackingParams), } diff --git a/polkadot/node/core/runtime-api/src/lib.rs b/polkadot/node/core/runtime-api/src/lib.rs index 78531d41272b..1d97e4676c58 100644 --- a/polkadot/node/core/runtime-api/src/lib.rs +++ b/polkadot/node/core/runtime-api/src/lib.rs @@ -162,6 +162,8 @@ where KeyOwnershipProof(relay_parent, validator_id, key_ownership_proof) => self .requests_cache .cache_key_ownership_proof((relay_parent, validator_id), key_ownership_proof), + RequestResult::ApprovalVotingParams(relay_parent, params) => + self.requests_cache.cache_approval_voting_params(relay_parent, params), SubmitReportDisputeLost(_, _, _, _) => {}, StagingParaBackingState(relay_parent, para_id, constraints) => self @@ -294,6 +296,8 @@ where Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) }, ), + Request::ApprovalVotingParams(sender) => query!(approval_voting_params(), sender) + .map(|sender| Request::ApprovalVotingParams(sender)), Request::StagingParaBackingState(para, sender) => query!(staging_para_backing_state(para), sender) @@ -545,6 +549,9 @@ where ver = Request::KEY_OWNERSHIP_PROOF_RUNTIME_REQUIREMENT, sender ), + Request::ApprovalVotingParams(sender) => { + query!(ApprovalVotingParams, approval_voting_params(), ver = 6, sender) + }, Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) => query!( SubmitReportDisputeLost, submit_report_dispute_lost(dispute_proof, key_ownership_proof), diff --git a/polkadot/node/core/runtime-api/src/tests.rs b/polkadot/node/core/runtime-api/src/tests.rs index c3f8108312be..d13e6cede356 100644 --- a/polkadot/node/core/runtime-api/src/tests.rs +++ b/polkadot/node/core/runtime-api/src/tests.rs @@ -20,12 +20,12 @@ use polkadot_node_primitives::{BabeAllowedSlots, BabeEpoch, BabeEpochConfigurati use polkadot_node_subsystem::SpawnGlue; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_primitives::{ - vstaging, AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, Slot, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, ValidatorSignature, + vstaging::{self, ApprovalVotingParams}, + AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + Slot, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_api::ApiError; use sp_core::testing::TaskExecutor; @@ -242,6 +242,11 @@ impl RuntimeApiSubsystemClient for MockSubsystemClient { todo!("Not required for tests") } + /// Approval voting configuration parameters + async fn approval_voting_params(&self, _: Hash) -> Result { + todo!("Not required for tests") + } + async fn current_epoch(&self, _: Hash) -> Result { Ok(self.babe_epoch.as_ref().unwrap().clone()) } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 746a4b4dab5c..3b8624e9166d 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -36,7 +36,10 @@ use polkadot_node_primitives::approval::{ v1::{ AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, }, - v2::{AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2}, + v2::{ + AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2, + IndirectSignedApprovalVoteV2, + }, }; use polkadot_node_subsystem::{ messages::{ @@ -120,7 +123,7 @@ struct ApprovalEntry { // The candidates claimed by the certificate. A mapping between bit index and candidate index. candidates: CandidateBitfield, // The approval signatures for each `CandidateIndex` claimed by the assignment certificate. - approvals: HashMap, + approvals: HashMap, // The validator index of the assignment signer. validator_index: ValidatorIndex, // Information required for gossiping to other peers using the grid topology. @@ -189,7 +192,8 @@ impl ApprovalEntry { // have received the approval. pub fn note_approval( &mut self, - approval: IndirectSignedApprovalVote, + approval: IndirectSignedApprovalVoteV2, + candidate_index: CandidateIndex, ) -> Result<(), ApprovalEntryError> { // First do some sanity checks: // - check validator index matches @@ -199,19 +203,18 @@ impl ApprovalEntry { return Err(ApprovalEntryError::InvalidValidatorIndex) } - if self.candidates.len() <= approval.candidate_index as usize { + if self.candidates.len() <= candidate_index as usize { return Err(ApprovalEntryError::CandidateIndexOutOfBounds) } - - if !self.candidates.bit_at(approval.candidate_index.as_bit_index()) { + if !self.candidates.bit_at((candidate_index).as_bit_index()) { return Err(ApprovalEntryError::InvalidCandidateIndex) } - if self.approvals.contains_key(&approval.candidate_index) { + if self.approvals.contains_key(&candidate_index) { return Err(ApprovalEntryError::DuplicateApproval) } - self.approvals.insert(approval.candidate_index, approval); + self.approvals.insert(candidate_index, approval.clone()); Ok(()) } @@ -221,12 +224,15 @@ impl ApprovalEntry { } // Get all approvals for all candidates claimed by the assignment. - pub fn approvals(&self) -> Vec { + pub fn approvals(&self) -> Vec { self.approvals.values().cloned().collect::>() } // Get the approval for a specific candidate index. - pub fn approval(&self, candidate_index: CandidateIndex) -> Option { + pub fn approval( + &self, + candidate_index: CandidateIndex, + ) -> Option { self.approvals.get(&candidate_index).cloned() } @@ -554,7 +560,7 @@ impl MessageSource { enum PendingMessage { Assignment(IndirectAssignmentCertV2, CandidateBitfield), - Approval(IndirectSignedApprovalVote), + Approval(IndirectSignedApprovalVoteV2), } #[overseer::contextbounds(ApprovalDistribution, prefix = self::overseer)] @@ -827,6 +833,48 @@ impl State { } } + async fn process_incoming_approvals( + &mut self, + ctx: &mut Context, + metrics: &Metrics, + peer_id: PeerId, + approvals: Vec, + ) { + gum::trace!( + target: LOG_TARGET, + peer_id = %peer_id, + num = approvals.len(), + "Processing approvals from a peer", + ); + for approval_vote in approvals.into_iter() { + if let Some(pending) = self.pending_known.get_mut(&approval_vote.block_hash) { + let block_hash = approval_vote.block_hash; + let validator_index = approval_vote.validator; + + gum::trace!( + target: LOG_TARGET, + %peer_id, + ?block_hash, + ?validator_index, + "Pending assignment candidates {:?}", + approval_vote.candidate_indices, + ); + + pending.push((peer_id, PendingMessage::Approval(approval_vote))); + + continue + } + + self.import_and_circulate_approval( + ctx, + metrics, + MessageSource::Peer(peer_id), + approval_vote, + ) + .await; + } + } + async fn process_incoming_peer_message( &mut self, ctx: &mut Context, @@ -884,42 +932,17 @@ impl State { }, Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Approvals( approvals, - )) | + )) => { + self.process_incoming_approvals(ctx, metrics, peer_id, approvals).await; + }, Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) => { - gum::trace!( - target: LOG_TARGET, - peer_id = %peer_id, - num = approvals.len(), - "Processing approvals from a peer", - ); - for approval_vote in approvals.into_iter() { - if let Some(pending) = self.pending_known.get_mut(&approval_vote.block_hash) { - let block_hash = approval_vote.block_hash; - let candidate_index = approval_vote.candidate_index; - let validator_index = approval_vote.validator; - - gum::trace!( - target: LOG_TARGET, - %peer_id, - ?block_hash, - ?candidate_index, - ?validator_index, - "Pending assignment", - ); - - pending.push((peer_id, PendingMessage::Approval(approval_vote))); - - continue - } - - self.import_and_circulate_approval( - ctx, - metrics, - MessageSource::Peer(peer_id), - approval_vote, - ) - .await; - } + self.process_incoming_approvals( + ctx, + metrics, + peer_id, + approvals.into_iter().map(|approval| approval.into()).collect::>(), + ) + .await; }, } } @@ -1065,8 +1088,11 @@ impl State { COST_UNEXPECTED_MESSAGE, ) .await; + gum::debug!(target: LOG_TARGET, "Received assignment for invalid block"); + metrics.on_assignment_recent_outdated(); } } + metrics.on_assignment_invalid_block(); return }, }; @@ -1099,6 +1125,7 @@ impl State { COST_DUPLICATE_MESSAGE, ) .await; + metrics.on_assignment_duplicate(); } else { gum::trace!( target: LOG_TARGET, @@ -1126,6 +1153,7 @@ impl State { COST_UNEXPECTED_MESSAGE, ) .await; + metrics.on_assignment_out_of_view(); }, } @@ -1142,6 +1170,7 @@ impl State { gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subject, "Known assignment"); peer_knowledge.received.insert(message_subject, message_kind); } + metrics.on_assignment_good_known(); return } @@ -1198,6 +1227,8 @@ impl State { ?peer_id, "Got an `AcceptedDuplicate` assignment", ); + metrics.on_assignment_duplicatevoting(); + return }, AssignmentCheckResult::TooFarInFuture => { @@ -1214,6 +1245,8 @@ impl State { COST_ASSIGNMENT_TOO_FAR_IN_THE_FUTURE, ) .await; + metrics.on_assignment_far(); + return }, AssignmentCheckResult::Bad(error) => { @@ -1231,6 +1264,7 @@ impl State { COST_INVALID_MESSAGE, ) .await; + metrics.on_assignment_bad(); return }, } @@ -1299,6 +1333,9 @@ impl State { continue } + if !topology.map(|topology| topology.is_validator(&peer)).unwrap_or(false) { + continue + } // Note: at this point, we haven't received the message from any peers // other than the source peer, and we just got it, so we haven't sent it // to any peers either. @@ -1340,12 +1377,95 @@ impl State { } } + async fn check_approval_can_be_processed( + ctx: &mut Context, + message_subjects: &Vec, + message_kind: MessageKind, + entry: &mut BlockEntry, + reputation: &mut ReputationAggregator, + peer_id: PeerId, + metrics: &Metrics, + ) -> bool { + for message_subject in message_subjects { + if !entry.knowledge.contains(&message_subject, MessageKind::Assignment) { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + ?message_subject, + "Unknown approval assignment", + ); + modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; + metrics.on_approval_unknown_assignment(); + return false + } + + // check if our knowledge of the peer already contains this approval + match entry.known_by.entry(peer_id) { + hash_map::Entry::Occupied(mut knowledge) => { + let peer_knowledge = knowledge.get_mut(); + if peer_knowledge.contains(&message_subject, message_kind) { + if !peer_knowledge.received.insert(message_subject.clone(), message_kind) { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + ?message_subject, + "Duplicate approval", + ); + + modify_reputation( + reputation, + ctx.sender(), + peer_id, + COST_DUPLICATE_MESSAGE, + ) + .await; + metrics.on_approval_duplicate(); + } + return false + } + }, + hash_map::Entry::Vacant(_) => { + gum::debug!( + target: LOG_TARGET, + ?peer_id, + ?message_subject, + "Approval from a peer is out of view", + ); + modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE) + .await; + metrics.on_approval_out_of_view(); + }, + } + } + + let good_known_approval = + message_subjects.iter().fold(false, |accumulator, message_subject| { + // if the approval is known to be valid, reward the peer + if entry.knowledge.contains(&message_subject, message_kind) { + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge.received.insert(message_subject.clone(), message_kind); + } + // We already processed this approval no need to continue. + true + } else { + accumulator + } + }); + if good_known_approval { + gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subjects, "Known approval"); + metrics.on_approval_good_known(); + modify_reputation(reputation, ctx.sender(), peer_id, BENEFIT_VALID_MESSAGE).await; + } + + !good_known_approval + } + async fn import_and_circulate_approval( &mut self, ctx: &mut Context, metrics: &Metrics, source: MessageSource, - vote: IndirectSignedApprovalVote, + vote: IndirectSignedApprovalVoteV2, ) { let _span = self .spans @@ -1364,10 +1484,21 @@ impl State { let block_hash = vote.block_hash; let validator_index = vote.validator; - let candidate_index = vote.candidate_index; - + let candidate_indices = &vote.candidate_indices; let entry = match self.blocks.get_mut(&block_hash) { - Some(entry) if entry.candidates.get(candidate_index as usize).is_some() => entry, + Some(entry) + if vote.candidate_indices.iter_ones().fold(true, |result, candidate_index| { + let approval_entry_exists = entry.candidates.get(candidate_index as usize).is_some(); + if !approval_entry_exists { + gum::debug!( + target: LOG_TARGET, ?block_hash, ?candidate_index, validator_index = ?vote.validator, candidate_indices = ?vote.candidate_indices, + peer_id = ?source.peer_id(), "Received approval before assignment" + ); + metrics.on_approval_entry_not_found(); + } + approval_entry_exists && result + }) => + entry, _ => { if let Some(peer_id) = source.peer_id() { if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { @@ -1376,7 +1507,7 @@ impl State { ?peer_id, ?block_hash, ?validator_index, - ?candidate_index, + ?candidate_indices, "Approval from a peer is out of view", ); modify_reputation( @@ -1386,6 +1517,10 @@ impl State { COST_UNEXPECTED_MESSAGE, ) .await; + metrics.on_approval_invalid_block(); + + } else { + metrics.on_approval_recent_outdated(); } } return @@ -1393,81 +1528,30 @@ impl State { }; // compute metadata on the assignment. - let message_subject = MessageSubject(block_hash, candidate_index.into(), validator_index); + let message_subjects = candidate_indices + .iter_ones() + .map(|candidate_index| { + MessageSubject( + block_hash, + (candidate_index as CandidateIndex).into(), + validator_index, + ) + }) + .collect_vec(); let message_kind = MessageKind::Approval; if let Some(peer_id) = source.peer_id() { - if !entry.knowledge.contains(&message_subject, MessageKind::Assignment) { - gum::debug!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Unknown approval assignment", - ); - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - COST_UNEXPECTED_MESSAGE, - ) - .await; - return - } - - // check if our knowledge of the peer already contains this approval - match entry.known_by.entry(peer_id) { - hash_map::Entry::Occupied(mut knowledge) => { - let peer_knowledge = knowledge.get_mut(); - if peer_knowledge.contains(&message_subject, message_kind) { - if !peer_knowledge.received.insert(message_subject.clone(), message_kind) { - gum::debug!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Duplicate approval", - ); - - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - COST_DUPLICATE_MESSAGE, - ) - .await; - } - return - } - }, - hash_map::Entry::Vacant(_) => { - gum::debug!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Approval from a peer is out of view", - ); - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - COST_UNEXPECTED_MESSAGE, - ) - .await; - }, - } - - // if the approval is known to be valid, reward the peer - if entry.knowledge.contains(&message_subject, message_kind) { - gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subject, "Known approval"); - modify_reputation( - &mut self.reputation, - ctx.sender(), - peer_id, - BENEFIT_VALID_MESSAGE, - ) - .await; - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); - } + if !Self::check_approval_can_be_processed( + ctx, + &message_subjects, + message_kind, + entry, + &mut self.reputation, + peer_id, + metrics, + ) + .await + { return } @@ -1489,7 +1573,6 @@ impl State { gum::trace!( target: LOG_TARGET, ?peer_id, - ?message_subject, ?result, "Checked approval", ); @@ -1503,9 +1586,11 @@ impl State { ) .await; - entry.knowledge.insert(message_subject.clone(), message_kind); - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); + for message_subject in &message_subjects { + entry.knowledge.insert(message_subject.clone(), message_kind); + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge.received.insert(message_subject.clone(), message_kind); + } } }, ApprovalCheckResult::Bad(error) => { @@ -1522,73 +1607,80 @@ impl State { %error, "Got a bad approval from peer", ); + metrics.on_approval_bad(); return }, } } else { - if !entry.knowledge.insert(message_subject.clone(), message_kind) { - // if we already imported an approval, there is no need to distribute it again + let all_approvals_imported_already = + message_subjects.iter().fold(true, |result, message_subject| { + !entry.knowledge.insert(message_subject.clone(), message_kind) && result + }); + if all_approvals_imported_already { + // if we already imported all approvals, there is no need to distribute it again gum::warn!( target: LOG_TARGET, - ?message_subject, "Importing locally an already known approval", ); return } else { gum::debug!( target: LOG_TARGET, - ?message_subject, "Importing locally a new approval", ); } } - let required_routing = match entry.approval_entry(candidate_index, validator_index) { - Some(approval_entry) => { - // Invariant: to our knowledge, none of the peers except for the `source` know about - // the approval. - metrics.on_approval_imported(); - - if let Err(err) = approval_entry.note_approval(vote.clone()) { - // this would indicate a bug in approval-voting: - // - validator index mismatch - // - candidate index mismatch - // - duplicate approval - gum::warn!( - target: LOG_TARGET, - hash = ?block_hash, - ?candidate_index, - ?validator_index, - ?err, - "Possible bug: Vote import failed", - ); + let mut required_routing = RequiredRouting::None; - return - } - - approval_entry.routing_info().required_routing - }, - None => { + for candidate_index in candidate_indices.iter_ones() { + // The entry is created when assignment is imported, so we assume this exists. + let approval_entry = entry.approval_entry(candidate_index as _, validator_index); + if approval_entry.is_none() { let peer_id = source.peer_id(); // This indicates a bug in approval-distribution, since we check the knowledge at // the begining of the function. gum::warn!( target: LOG_TARGET, ?peer_id, - ?message_subject, "Unknown approval assignment", ); // No rep change as this is caused by an issue + metrics.on_approval_unexpected(); return - }, - }; + } + + let approval_entry = approval_entry.expect("Just checked above; qed"); + + if let Err(err) = approval_entry.note_approval(vote.clone(), candidate_index as _) { + // this would indicate a bug in approval-voting: + // - validator index mismatch + // - candidate index mismatch + // - duplicate approval + gum::warn!( + target: LOG_TARGET, + hash = ?block_hash, + ?candidate_index, + ?validator_index, + ?err, + "Possible bug: Vote import failed", + ); + metrics.on_approval_bug(); + return + } + required_routing = approval_entry.routing_info().required_routing; + } + + // Invariant: to our knowledge, none of the peers except for the `source` know about the + // approval. + metrics.on_approval_imported(); // Dispatch a ApprovalDistributionV1Message::Approval(vote) // to all peers required by the topology, with the exception of the source peer. let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); - let message_subject = &message_subject; + let message_subjects_clone = message_subjects.clone(); let peer_filter = move |peer, knowledge: &PeerKnowledge| { if Some(peer) == source_peer.as_ref() { return false @@ -1605,7 +1697,10 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || knowledge.sent.contains(message_subject, MessageKind::Assignment) + in_topology || + message_subjects_clone.iter().fold(true, |result, message_subject| { + result && knowledge.sent.contains(message_subject, MessageKind::Assignment) + }) }; let peers = entry @@ -1619,7 +1714,9 @@ impl State { for peer in peers.iter() { // we already filtered peers above, so this should always be Some if let Some(entry) = entry.known_by.get_mut(&peer.0) { - entry.sent.insert(message_subject.clone(), message_kind); + for message_subject in &message_subjects { + entry.sent.insert(message_subject.clone(), message_kind); + } } } @@ -1628,7 +1725,6 @@ impl State { gum::trace!( target: LOG_TARGET, ?block_hash, - ?candidate_index, local = source.peer_id().is_none(), num_peers = peers.len(), "Sending an approval to peers", @@ -1641,10 +1737,22 @@ impl State { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( v1_peers, Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(approvals.clone()), + protocol_v1::ApprovalDistributionMessage::Approvals( + approvals + .clone() + .into_iter() + .filter(|approval| approval.candidate_indices.count_ones() == 1) + .map(|approval| { + approval + .try_into() + .expect("We checked len() == 1 so it should not fail; qed") + }) + .collect::>(), + ), )), )) .await; + metrics.on_approval_sent_v1(); } if !v2_peers.is_empty() { @@ -1657,6 +1765,7 @@ impl State { ), )) .await; + metrics.on_approval_sent_v2(); } } } @@ -1665,7 +1774,7 @@ impl State { fn get_approval_signatures( &mut self, indices: HashSet<(Hash, CandidateIndex)>, - ) -> HashMap { + ) -> HashMap, ValidatorSignature)> { let mut all_sigs = HashMap::new(); for (hash, index) in indices { let _span = self @@ -1692,8 +1801,22 @@ impl State { .approval_entries(index) .into_iter() .filter_map(|approval_entry| approval_entry.approval(index)) - .map(|approval| (approval.validator, approval.signature)) - .collect::>(); + .map(|approval| { + ( + approval.validator, + ( + hash, + approval + .candidate_indices + .iter_ones() + .map(|val| val as CandidateIndex) + .collect_vec(), + approval.signature, + ), + ) + }) + .collect::, ValidatorSignature)>>( + ); all_sigs.extend(sigs); } all_sigs @@ -1714,7 +1837,7 @@ impl State { let _timer = metrics.time_unify_with_peer(); let mut assignments_to_send = Vec::new(); - let mut approvals_to_send = Vec::new(); + let mut approvals_to_send = HashMap::new(); let view_finalized_number = view.finalized_number; for head in view.into_iter() { @@ -1751,6 +1874,13 @@ impl State { t.local_grid_neighbors().route_to_peer(required_routing, peer_id) }); in_topology || { + if !topology + .map(|topology| topology.is_validator(peer_id)) + .unwrap_or(false) + { + return false + } + let route_random = random_routing.sample(total_peers, rng); if route_random { random_routing.inc_sent(); @@ -1778,12 +1908,51 @@ impl State { // Filter approval votes. for approval_message in approval_messages { - let (approval_knowledge, message_kind) = approval_entry - .create_approval_knowledge(block, approval_message.candidate_index); + let (should_forward_approval, candidates_covered_by_approvals) = + approval_message.candidate_indices.iter_ones().fold( + (true, Vec::new()), + |(should_forward_approval, mut new_covered_approvals), + approval_candidate_index| { + let (message_subject, message_kind) = approval_entry + .create_approval_knowledge( + block, + approval_candidate_index as _, + ); + // The assignments for all candidates signed in the approval + // should already have been sent to the peer, otherwise we can't + // send our approval and risk breaking our reputation. + let should_forward_approval = should_forward_approval && + peer_knowledge + .contains(&message_subject, MessageKind::Assignment); + if !peer_knowledge.contains(&message_subject, message_kind) { + new_covered_approvals.push((message_subject, message_kind)) + } + + (should_forward_approval, new_covered_approvals) + }, + ); - if !peer_knowledge.contains(&approval_knowledge, message_kind) { - peer_knowledge.sent.insert(approval_knowledge, message_kind); - approvals_to_send.push(approval_message); + if should_forward_approval { + if !approvals_to_send.contains_key(&( + approval_message.block_hash, + approval_message.validator, + approval_message.candidate_indices.clone(), + )) { + approvals_to_send.insert( + ( + approval_message.block_hash, + approval_message.validator, + approval_message.candidate_indices.clone(), + ), + approval_message, + ); + } + + candidates_covered_by_approvals.into_iter().for_each( + |(approval_knowledge, message_kind)| { + peer_knowledge.sent.insert(approval_knowledge, message_kind); + }, + ); } } } @@ -1818,8 +1987,12 @@ impl State { "Sending approvals to unified peer", ); - send_approvals_batched(sender, approvals_to_send, &vec![(peer_id, protocol_version)]) - .await; + send_approvals_batched( + sender, + approvals_to_send.into_values().collect_vec(), + &vec![(peer_id, protocol_version)], + ) + .await; } } @@ -1956,6 +2129,7 @@ impl State { // Punish the peer for the invalid message. modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) .await; + gum::error!(target: LOG_TARGET, block_hash = ?cert.block_hash, ?candidate_index, validator_index = ?cert.validator, kind = ?cert.cert.kind, "Bad assignment v1"); } else { sanitized_assignments.push((cert.into(), candidate_index.into())) } @@ -1998,6 +2172,9 @@ impl State { // Punish the peer for the invalid message. modify_reputation(&mut self.reputation, sender, peer_id, COST_OVERSIZED_BITFIELD) .await; + for candidate_index in candidate_bitfield.iter_ones() { + gum::error!(target: LOG_TARGET, block_hash = ?cert.block_hash, ?candidate_index, validator_index = ?cert.validator, "Bad assignment v2"); + } } else { sanitized_assignments.push((cert, candidate_bitfield)) } @@ -2085,15 +2262,29 @@ async fn adjust_required_routing_and_propagate { gum::debug!( target: LOG_TARGET, - "Distributing our approval vote on candidate (block={}, index={})", + "Distributing our approval vote on candidate (block={}, index={:?})", vote.block_hash, - vote.candidate_index, + vote.candidate_indices, ); state @@ -2315,7 +2506,7 @@ pub const MAX_ASSIGNMENT_BATCH_SIZE: usize = ensure_size_not_zero( /// The maximum amount of approvals per batch is 33% of maximum allowed by protocol. pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( - MAX_NOTIFICATION_SIZE as usize / std::mem::size_of::() / 3, + MAX_NOTIFICATION_SIZE as usize / std::mem::size_of::() / 3, ); // Low level helper for sending assignments. @@ -2409,14 +2600,19 @@ pub(crate) async fn send_assignments_batched( /// Send approvals while honoring the `max_notification_size` of the protocol and peer version. pub(crate) async fn send_approvals_batched( sender: &mut impl overseer::ApprovalDistributionSenderTrait, - approvals: impl IntoIterator + Clone, + approvals: impl IntoIterator + Clone, peers: &[(PeerId, ProtocolVersion)], ) { let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); let v2_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); if !v1_peers.is_empty() { - let mut batches = approvals.clone().into_iter().peekable(); + let mut batches = approvals + .clone() + .into_iter() + .filter(|approval| approval.candidate_indices.count_ones() == 1) + .map(|val| val.try_into().expect("We checked conversion should succeed; qed")) + .peekable(); while batches.peek().is_some() { let batch: Vec<_> = batches.by_ref().take(MAX_APPROVAL_BATCH_SIZE).collect(); diff --git a/polkadot/node/network/approval-distribution/src/metrics.rs b/polkadot/node/network/approval-distribution/src/metrics.rs index 6864259e6fdb..094874820f26 100644 --- a/polkadot/node/network/approval-distribution/src/metrics.rs +++ b/polkadot/node/network/approval-distribution/src/metrics.rs @@ -31,6 +31,8 @@ struct MetricsInner { time_unify_with_peer: prometheus::Histogram, time_import_pending_now_known: prometheus::Histogram, time_awaiting_approval_voting: prometheus::Histogram, + assignments_received_result: prometheus::CounterVec, + approvals_received_result: prometheus::CounterVec, } trait AsLabel { @@ -78,6 +80,144 @@ impl Metrics { .map(|metrics| metrics.time_import_pending_now_known.start_timer()) } + pub fn on_approval_already_known(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["known"]).inc() + } + } + + pub fn on_approval_entry_not_found(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["noapprovalentry"]).inc() + } + } + + pub fn on_approval_recent_outdated(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["outdated"]).inc() + } + } + + pub fn on_approval_invalid_block(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["invalidblock"]).inc() + } + } + + pub fn on_approval_unknown_assignment(&self) { + if let Some(metrics) = &self.0 { + metrics + .approvals_received_result + .with_label_values(&["unknownassignment"]) + .inc() + } + } + + pub fn on_approval_duplicate(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["duplicate"]).inc() + } + } + + pub fn on_approval_out_of_view(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["outofview"]).inc() + } + } + + pub fn on_approval_good_known(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["goodknown"]).inc() + } + } + + pub fn on_approval_bad(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["bad"]).inc() + } + } + + pub fn on_approval_unexpected(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["unexpected"]).inc() + } + } + + pub fn on_approval_bug(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["bug"]).inc() + } + } + + pub fn on_approval_sent_v1(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["v1"]).inc() + } + } + + pub fn on_approval_sent_v2(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_received_result.with_label_values(&["v2"]).inc() + } + } + + pub fn on_assignment_already_known(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["known"]).inc() + } + } + + pub fn on_assignment_recent_outdated(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["outdated"]).inc() + } + } + + pub fn on_assignment_invalid_block(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["invalidblock"]).inc() + } + } + + pub fn on_assignment_duplicate(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["duplicate"]).inc() + } + } + + pub fn on_assignment_out_of_view(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["outofview"]).inc() + } + } + + pub fn on_assignment_good_known(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["goodknown"]).inc() + } + } + + pub fn on_assignment_bad(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["bad"]).inc() + } + } + + pub fn on_assignment_duplicatevoting(&self) { + if let Some(metrics) = &self.0 { + metrics + .assignments_received_result + .with_label_values(&["duplicatevoting"]) + .inc() + } + } + + pub fn on_assignment_far(&self) { + if let Some(metrics) = &self.0 { + metrics.assignments_received_result.with_label_values(&["far"]).inc() + } + } + pub(crate) fn time_awaiting_approval_voting( &self, ) -> Option { @@ -167,6 +307,26 @@ impl MetricsTrait for Metrics { ).buckets(vec![0.0001, 0.0004, 0.0016, 0.0064, 0.0256, 0.1024, 0.4096, 1.6384, 3.2768, 4.9152, 6.5536,]))?, registry, )?, + assignments_received_result: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_assignments_received_result", + "Result of a processed assignement", + ), + &["status"] + )?, + registry, + )?, + approvals_received_result: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_approvals_received_result", + "Result of a processed approval", + ), + &["status"] + )?, + registry, + )?, }; Ok(Metrics(Some(metrics))) } diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index f0c3c4f8ba64..b9fc5baa820c 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -133,14 +133,13 @@ fn make_gossip_topology( all_peers: &[(PeerId, AuthorityDiscoveryId)], neighbors_x: &[usize], neighbors_y: &[usize], + local_index: usize, ) -> network_bridge_event::NewGossipTopology { // This builds a grid topology which is a square matrix. // The local validator occupies the top left-hand corner. // The X peers occupy the same row and the Y peers occupy // the same column. - let local_index = 1; - assert_eq!( neighbors_x.len(), neighbors_y.len(), @@ -365,10 +364,11 @@ fn state_with_reputation_delay() -> State { /// the new peer sends us the same assignment #[test] fn try_import_the_same_assignment() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); - let peer_d = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let peer_d = peers.get(4).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); @@ -379,6 +379,10 @@ fn try_import_the_same_assignment() { setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; + // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under + // testing. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { hash, @@ -449,10 +453,11 @@ fn try_import_the_same_assignment() { /// cores. #[test] fn try_import_the_same_assignment_v2() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); - let peer_d = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let peer_d = peers.get(4).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); @@ -463,6 +468,10 @@ fn try_import_the_same_assignment_v2() { setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under + // testing. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { hash, @@ -690,14 +699,19 @@ fn spam_attack_results_in_negative_reputation_change() { #[test] fn peer_sending_us_the_same_we_just_sent_them_is_ok() { let parent_hash = Hash::repeat_byte(0xFF); - let peer_a = PeerId::random(); let hash = Hash::repeat_byte(0xAA); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; let peer = &peer_a; setup_peer_with_view(overseer, peer, view![], ValidationVersion::V1).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + // new block `hash` with 1 candidates let meta = BlockApprovalMeta { hash, @@ -766,9 +780,11 @@ fn peer_sending_us_the_same_we_just_sent_them_is_ok() { #[test] fn import_approval_happy_path() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); @@ -791,6 +807,9 @@ fn import_approval_happy_path() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; + // Set up a gossip topology, where a, b, and c are topology neighboors to the node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + // import an assignment related to `hash` locally let validator_index = ValidatorIndex(0); let candidate_index = 0u32; @@ -833,14 +852,14 @@ fn import_approval_happy_path() { ); // send the an approval from peer_b - let approval = IndirectSignedApprovalVote { + let approval = IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index, + candidate_indices: candidate_index.into(), validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -901,14 +920,14 @@ fn import_approval_bad() { let cert = fake_assignment_cert(hash, validator_index); // send the an approval from peer_b, we don't have an assignment yet - let approval = IndirectSignedApprovalVote { + let approval = IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index, + candidate_indices: candidate_index.into(), validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; @@ -933,8 +952,8 @@ fn import_approval_bad() { expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; // and try again - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, &peer_b, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1033,7 +1052,8 @@ fn update_peer_view() { let hash_b = Hash::repeat_byte(0xBB); let hash_c = Hash::repeat_byte(0xCC); let hash_d = Hash::repeat_byte(0xDD); - let peer_a = PeerId::random(); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; let peer = &peer_a; let state = test_harness(State::default(), |mut virtual_overseer| async move { @@ -1067,6 +1087,9 @@ fn update_peer_view() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta_a, meta_b, meta_c]); overseer_send(overseer, msg).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + let cert_a = fake_assignment_cert(hash_a, ValidatorIndex(0)); let cert_b = fake_assignment_cert(hash_b, ValidatorIndex(0)); @@ -1249,14 +1272,14 @@ fn import_remotely_then_locally() { assert!(overseer.recv().timeout(TIMEOUT).await.is_none(), "no message should be sent"); // send the approval remotely - let approval = IndirectSignedApprovalVote { + let approval = IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index, + candidate_indices: candidate_index.into(), validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_v1::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer(overseer, peer, msg).await; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1280,7 +1303,8 @@ fn import_remotely_then_locally() { #[test] fn sends_assignments_even_when_state_is_approved() { - let peer_a = PeerId::random(); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let peer = &peer_a; @@ -1300,6 +1324,9 @@ fn sends_assignments_even_when_state_is_approved() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + let validator_index = ValidatorIndex(0); let candidate_index = 0u32; @@ -1321,8 +1348,11 @@ fn sends_assignments_even_when_state_is_approved() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; // connect the peer. setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V1).await; @@ -1365,7 +1395,8 @@ fn sends_assignments_even_when_state_is_approved() { /// assignemnts. #[test] fn sends_assignments_even_when_state_is_approved_v2() { - let peer_a = PeerId::random(); + let peers = make_peers_and_authority_ids(8); + let peer_a = peers.first().unwrap().0; let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); let peer = &peer_a; @@ -1385,6 +1416,9 @@ fn sends_assignments_even_when_state_is_approved_v2() { let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; + // Setup a topology where peer_a is neigboor to current node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0], &[2], 1)).await; + let validator_index = ValidatorIndex(0); let cores = vec![0, 1, 2, 3]; let candidate_bitfield: CandidateBitfield = cores.clone().try_into().unwrap(); @@ -1401,9 +1435,9 @@ fn sends_assignments_even_when_state_is_approved_v2() { // Assumes candidate index == core index. let approvals = cores .iter() - .map(|core| IndirectSignedApprovalVote { + .map(|core| IndirectSignedApprovalVoteV2 { block_hash: hash, - candidate_index: *core, + candidate_indices: (*core).into(), validator: validator_index, signature: dummy_signature(), }) @@ -1454,8 +1488,8 @@ fn sends_assignments_even_when_state_is_approved_v2() { )) => { // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a // hashmap as well. - let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); - let approvals = approvals.into_iter().map(|approval| (approval.candidate_index, approval)).collect::>(); + let sent_approvals = sent_approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); + let approvals = approvals.into_iter().map(|approval| (approval.candidate_indices.clone(), approval)).collect::>(); assert_eq!(peers, vec![*peer]); assert_eq!(sent_approvals, approvals); @@ -1565,13 +1599,19 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology( + 1, + &peers, + &[0, 10, 20, 30, 40, 60, 70, 80], + &[50, 51, 52, 53, 54, 55, 56, 57], + 1, + ), ) .await; let expected_indices = [ // Both dimensions in the gossip topology - 0, 10, 20, 30, 50, 51, 52, 53, + 0, 10, 20, 30, 40, 60, 70, 80, 50, 51, 52, 53, 54, 55, 56, 57, ]; // new block `hash_a` with 1 candidates @@ -1608,8 +1648,11 @@ fn propagates_locally_generated_assignment_to_both_dimensions() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1673,7 +1716,7 @@ fn propagates_assignments_along_unshared_dimension() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -1816,13 +1859,19 @@ fn propagates_to_required_after_connect() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology( + 1, + &peers, + &[0, 10, 20, 30, 40, 60, 70, 80], + &[50, 51, 52, 53, 54, 55, 56, 57], + 1, + ), ) .await; let expected_indices = [ // Both dimensions in the gossip topology, minus omitted. - 20, 30, 52, 53, + 20, 30, 40, 60, 70, 80, 52, 53, 54, 55, 56, 57, ]; // new block `hash_a` with 1 candidates @@ -1859,8 +1908,11 @@ fn propagates_to_required_after_connect() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -1987,53 +2039,21 @@ fn sends_to_more_peers_after_getting_topology() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; - let mut expected_indices = vec![0, 10, 20, 30, 50, 51, 52, 53]; - let assignment_sent_peers = assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Assignments(sent_assignments) - )) - )) => { - // Only sends to random peers. - assert_eq!(sent_peers.len(), 4); - for peer in &sent_peers { - let i = peers.iter().position(|p| peer == &p.0).unwrap(); - // Random gossip before topology can send to topology-targeted peers. - // Remove them from the expected indices so we don't expect - // them to get the messages again after the assignment. - expected_indices.retain(|&i2| i2 != i); - } - assert_eq!(sent_assignments, assignments); - sent_peers - } - ); - - assert_matches!( - overseer_recv(overseer).await, - AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( - sent_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals(sent_approvals) - )) - )) => { - // Random sampling is reused from the assignment. - assert_eq!(sent_peers, assignment_sent_peers); - assert_eq!(sent_approvals, approvals); - } - ); + let expected_indices = vec![0, 10, 20, 30, 50, 51, 52, 53]; // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2136,7 +2156,7 @@ fn originator_aggression_l1() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2149,8 +2169,11 @@ fn originator_aggression_l1() { ) .await; - overseer_send(overseer, ApprovalDistributionMessage::DistributeApproval(approval.clone())) - .await; + overseer_send( + overseer, + ApprovalDistributionMessage::DistributeApproval(approval.clone().into()), + ) + .await; let assignments = vec![(cert.clone(), candidate_index)]; let approvals = vec![approval.clone()]; @@ -2292,7 +2315,7 @@ fn non_originator_aggression_l1() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2397,7 +2420,7 @@ fn non_originator_aggression_l2() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2543,7 +2566,7 @@ fn resends_messages_periodically() { // Set up a gossip topology. setup_gossip_topology( overseer, - make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53]), + make_gossip_topology(1, &peers, &[0, 10, 20, 30], &[50, 51, 52, 53], 1), ) .await; @@ -2686,9 +2709,9 @@ fn batch_test_round(message_count: usize) { .collect(); let approvals: Vec<_> = validators - .map(|index| IndirectSignedApprovalVote { + .map(|index| IndirectSignedApprovalVoteV2 { block_hash: Hash::zero(), - candidate_index: 0, + candidate_indices: 0u32.into(), validator: ValidatorIndex(index as u32), signature: dummy_signature(), }) @@ -2755,7 +2778,7 @@ fn batch_test_round(message_count: usize) { assert_eq!(peers.len(), 1); for (message_index, approval) in sent_approvals.iter().enumerate() { - assert_eq!(approval, &approvals[approval_index + message_index]); + assert_eq!(approval, &approvals[approval_index + message_index].clone().try_into().unwrap()); } } ); diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index fb0cdb720571..c8a628645f34 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -104,7 +104,8 @@ const TIMEOUT_START_NEW_REQUESTS: Duration = CHUNK_REQUEST_TIMEOUT; const TIMEOUT_START_NEW_REQUESTS: Duration = Duration::from_millis(100); /// PoV size limit in bytes for which prefer fetching from backers. -const SMALL_POV_LIMIT: usize = 128 * 1024; +//TODO: Cleanup increased for versi testing +const SMALL_POV_LIMIT: usize = 3 * 1024 * 1024; #[derive(Clone, PartialEq)] /// The strategy we use to recover the PoV. diff --git a/polkadot/node/network/protocol/src/grid_topology.rs b/polkadot/node/network/protocol/src/grid_topology.rs index 99dd513c4d79..8bd9adbc17c1 100644 --- a/polkadot/node/network/protocol/src/grid_topology.rs +++ b/polkadot/node/network/protocol/src/grid_topology.rs @@ -73,12 +73,20 @@ pub struct SessionGridTopology { shuffled_indices: Vec, /// The canonical shuffling of validators for the session. canonical_shuffling: Vec, + /// The list of peer-ids in an efficient way to search. + peer_ids: HashSet, } impl SessionGridTopology { /// Create a new session grid topology. pub fn new(shuffled_indices: Vec, canonical_shuffling: Vec) -> Self { - SessionGridTopology { shuffled_indices, canonical_shuffling } + let mut peer_ids = HashSet::new(); + for peer_info in canonical_shuffling.iter() { + for peer_id in peer_info.peer_ids.iter() { + peer_ids.insert(*peer_id); + } + } + SessionGridTopology { shuffled_indices, canonical_shuffling, peer_ids } } /// Produces the outgoing routing logic for a particular peer. @@ -111,6 +119,11 @@ impl SessionGridTopology { Some(grid_subset) } + + /// Tells if a given peer id is validator in a session + pub fn is_validator(&self, peer: &PeerId) -> bool { + self.peer_ids.contains(peer) + } } struct MatrixNeighbors { @@ -273,6 +286,11 @@ impl SessionGridTopologyEntry { pub fn get(&self) -> &SessionGridTopology { &self.topology } + + /// Tells if a given peer id is validator in a session + pub fn is_validator(&self, peer: &PeerId) -> bool { + self.topology.is_validator(peer) + } } /// A set of topologies indexed by session @@ -347,6 +365,7 @@ impl Default for SessionBoundGridTopologyStorage { topology: SessionGridTopology { shuffled_indices: Vec::new(), canonical_shuffling: Vec::new(), + peer_ids: Default::default(), }, local_neighbors: GridNeighbors::empty(), }, diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index ba9b9a7f4900..8cd11bafa5f6 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -599,10 +599,7 @@ pub mod vstaging { }; use polkadot_node_primitives::{ - approval::{ - v1::IndirectSignedApprovalVote, - v2::{CandidateBitfield, IndirectAssignmentCertV2}, - }, + approval::v2::{CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, UncheckedSignedFullStatement, }; @@ -780,7 +777,7 @@ pub mod vstaging { Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), /// Approvals for candidates in some recent, unfinalized block. #[codec(index = 1)] - Approvals(Vec), + Approvals(Vec), } /// Dummy network message type, so we will receive connect/disconnect events. diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval.rs index bfa1f1aa47d2..49635023cd56 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval.rs @@ -219,7 +219,9 @@ pub mod v2 { use std::ops::BitOr; use bitvec::{prelude::Lsb0, vec::BitVec}; - use polkadot_primitives::{CandidateIndex, CoreIndex, Hash, ValidatorIndex}; + use polkadot_primitives::{ + CandidateIndex, CoreIndex, Hash, ValidatorIndex, ValidatorSignature, + }; /// A static context associated with producing randomness for a core. pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE v2"; @@ -473,6 +475,59 @@ pub mod v2 { }) } } + + impl From for IndirectSignedApprovalVoteV2 { + fn from(value: super::v1::IndirectSignedApprovalVote) -> Self { + Self { + block_hash: value.block_hash, + validator: value.validator, + candidate_indices: value.candidate_index.into(), + signature: value.signature, + } + } + } + + /// Errors that can occur when trying to convert to/from approvals v1/v2 + #[derive(Debug)] + pub enum ApprovalConversionError { + /// More than one candidate was signed. + MoreThanOneCandidate(usize), + } + + impl TryFrom for super::v1::IndirectSignedApprovalVote { + type Error = ApprovalConversionError; + + fn try_from(value: IndirectSignedApprovalVoteV2) -> Result { + if value.candidate_indices.count_ones() != 1 { + return Err(ApprovalConversionError::MoreThanOneCandidate( + value.candidate_indices.count_ones(), + )) + } + Ok(Self { + block_hash: value.block_hash, + validator: value.validator, + candidate_index: value.candidate_indices.first_one().expect("Qed we checked above") + as u32, + signature: value.signature, + }) + } + } + + /// A signed approval vote which references the candidate indirectly via the block. + /// + /// In practice, we have a look-up from block hash and candidate index to candidate hash, + /// so this can be transformed into a `SignedApprovalVote`. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct IndirectSignedApprovalVoteV2 { + /// A block hash where the candidate appears. + pub block_hash: Hash, + /// The index of the candidate in the list of candidates fully included as-of the block. + pub candidate_indices: CandidateBitfield, + /// The validator index. + pub validator: ValidatorIndex, + /// The signature by the validator. + pub signature: ValidatorSignature, + } } #[cfg(test)] diff --git a/polkadot/node/primitives/src/disputes/message.rs b/polkadot/node/primitives/src/disputes/message.rs index 89d3ea6c0af9..31fe73a7ba1c 100644 --- a/polkadot/node/primitives/src/disputes/message.rs +++ b/polkadot/node/primitives/src/disputes/message.rs @@ -170,7 +170,7 @@ impl DisputeMessage { let valid_vote = ValidDisputeVote { validator_index: valid_index, signature: valid_statement.validator_signature().clone(), - kind: *valid_kind, + kind: valid_kind.clone(), }; let invalid_vote = InvalidDisputeVote { diff --git a/polkadot/node/primitives/src/disputes/mod.rs b/polkadot/node/primitives/src/disputes/mod.rs index ae8602dd5fc4..c267445ad0be 100644 --- a/polkadot/node/primitives/src/disputes/mod.rs +++ b/polkadot/node/primitives/src/disputes/mod.rs @@ -46,6 +46,15 @@ pub struct SignedDisputeStatement { session_index: SessionIndex, } +/// Errors encountered while signing a dispute statement +#[derive(Debug)] +pub enum SignedDisputeStatementError { + /// Encountered a keystore error while signing + KeyStoreError(KeystoreError), + /// Could not generate signing payload + PayloadError, +} + /// Tracked votes on candidates, for the purposes of dispute resolution. #[derive(Debug, Clone)] pub struct CandidateVotes { @@ -107,8 +116,9 @@ impl ValidCandidateVotes { ValidDisputeStatementKind::BackingValid(_) | ValidDisputeStatementKind::BackingSeconded(_) => false, ValidDisputeStatementKind::Explicit | - ValidDisputeStatementKind::ApprovalChecking => { - occupied.insert((kind, sig)); + ValidDisputeStatementKind::ApprovalChecking | + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(_) => { + occupied.insert((kind.clone(), sig)); kind != occupied.get().0 }, }, @@ -213,16 +223,19 @@ impl SignedDisputeStatement { candidate_hash: CandidateHash, session_index: SessionIndex, validator_public: ValidatorId, - ) -> Result, KeystoreError> { + ) -> Result, SignedDisputeStatementError> { let dispute_statement = if valid { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) } else { DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) }; - let data = dispute_statement.payload_data(candidate_hash, session_index); + let data = dispute_statement + .payload_data(candidate_hash, session_index) + .map_err(|_| SignedDisputeStatementError::PayloadError)?; let signature = keystore - .sr25519_sign(ValidatorId::ID, validator_public.as_ref(), &data)? + .sr25519_sign(ValidatorId::ID, validator_public.as_ref(), &data) + .map_err(SignedDisputeStatementError::KeyStoreError)? .map(|sig| Self { dispute_statement, candidate_hash, diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs index d9553afa024b..f2f2bdefae7a 100644 --- a/polkadot/node/service/src/fake_runtime_api.rs +++ b/polkadot/node/service/src/fake_runtime_api.rs @@ -23,12 +23,12 @@ use beefy_primitives::ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefyS use grandpa_primitives::AuthorityId as GrandpaId; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use polkadot_primitives::{ - runtime_api, slashing, AccountId, AuthorityDiscoveryId, Balance, Block, BlockNumber, - CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, - DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Nonce, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, - ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, - ValidatorId, ValidatorIndex, ValidatorSignature, + runtime_api, slashing, vstaging::ApprovalVotingParams, AccountId, AuthorityDiscoveryId, + Balance, Block, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Nonce, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_core::OpaqueMetadata; use sp_runtime::{ @@ -116,6 +116,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl runtime_api::ParachainHost for Runtime { fn validators() -> Vec { unimplemented!() diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index f561a0e28512..d8fe54f98312 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -33,8 +33,8 @@ use polkadot_node_network_protocol::{ }; use polkadot_node_primitives::{ approval::{ - v1::{BlockApprovalMeta, IndirectSignedApprovalVote}, - v2::{CandidateBitfield, IndirectAssignmentCertV2}, + v1::BlockApprovalMeta, + v2::{CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2}, }, AvailableData, BabeEpoch, BlockWeight, CandidateVotes, CollationGenerationConfig, CollationSecondedSignal, DisputeMessage, DisputeStatus, ErasureChunk, PoV, @@ -42,14 +42,14 @@ use polkadot_node_primitives::{ ValidationResult, }; use polkadot_primitives::{ - slashing, vstaging as vstaging_primitives, AuthorityDiscoveryId, BackedCandidate, BlockNumber, - CandidateEvent, CandidateHash, CandidateIndex, CandidateReceipt, CollatorId, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupIndex, - GroupRotationInfo, Hash, Header as BlockHeader, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, MultiDisputeStatementSet, OccupiedCoreAssumption, PersistedValidationData, - PvfCheckStatement, PvfExecTimeoutKind, SessionIndex, SessionInfo, SignedAvailabilityBitfield, - SignedAvailabilityBitfields, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + slashing, vstaging as vstaging_primitives, vstaging::ApprovalVotingParams, + AuthorityDiscoveryId, BackedCandidate, BlockNumber, CandidateEvent, CandidateHash, + CandidateIndex, CandidateReceipt, CollatorId, CommittedCandidateReceipt, CoreState, + DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Header as BlockHeader, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, MultiDisputeStatementSet, + OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, PvfExecTimeoutKind, + SessionIndex, SessionInfo, SignedAvailabilityBitfield, SignedAvailabilityBitfields, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use polkadot_statement_table::v2::Misbehavior; use std::{ @@ -694,6 +694,8 @@ pub enum RuntimeApiRequest { slashing::OpaqueKeyOwnershipProof, RuntimeApiSender>, ), + /// Approval voting params + ApprovalVotingParams(RuntimeApiSender), /// Get the backing state of the given para. /// This is a staging API that will not be available on production runtimes. @@ -911,7 +913,7 @@ pub enum ApprovalVotingMessage { /// protocol. /// /// Should not be sent unless the block hash within the indirect vote is known. - CheckAndImportApproval(IndirectSignedApprovalVote, oneshot::Sender), + CheckAndImportApproval(IndirectSignedApprovalVoteV2, oneshot::Sender), /// Returns the highest possible ancestor hash of the provided block hash which is /// acceptable to vote on finality for. /// The `BlockNumber` provided is the number of the block's ancestor which is the @@ -927,7 +929,7 @@ pub enum ApprovalVotingMessage { /// requires calling into `approval-distribution`: Calls should be infrequent and bounded. GetApprovalSignaturesForCandidate( CandidateHash, - oneshot::Sender>, + oneshot::Sender, ValidatorSignature)>>, ), } @@ -943,7 +945,7 @@ pub enum ApprovalDistributionMessage { /// Distribute an approval vote for the local validator. The approval vote is assumed to be /// valid, relevant, and the corresponding approval already issued. /// If not, the subsystem is free to drop the message. - DistributeApproval(IndirectSignedApprovalVote), + DistributeApproval(IndirectSignedApprovalVoteV2), /// An update from the network bridge. #[from] NetworkBridgeUpdate(NetworkBridgeEvent), @@ -951,7 +953,7 @@ pub enum ApprovalDistributionMessage { /// Get all approval signatures for all chains a candidate appeared in. GetApprovalSignatures( HashSet<(Hash, CandidateIndex)>, - oneshot::Sender>, + oneshot::Sender, ValidatorSignature)>>, ), /// Approval checking lag update measured in blocks. ApprovalCheckingLagUpdate(BlockNumber), diff --git a/polkadot/node/subsystem-types/src/runtime_client.rs b/polkadot/node/subsystem-types/src/runtime_client.rs index 312cc4eec6ce..3e439dd4732d 100644 --- a/polkadot/node/subsystem-types/src/runtime_client.rs +++ b/polkadot/node/subsystem-types/src/runtime_client.rs @@ -16,12 +16,13 @@ use async_trait::async_trait; use polkadot_primitives::{ - runtime_api::ParachainHost, vstaging, Block, BlockNumber, CandidateCommitments, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Hash, Id, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + runtime_api::ParachainHost, + vstaging::{self, ApprovalVotingParams}, + Block, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sp_api::{ApiError, ApiExt, ProvideRuntimeApi}; @@ -247,6 +248,9 @@ pub trait RuntimeApiSubsystemClient { at: Hash, para_id: Id, ) -> Result, ApiError>; + + /// Approval voting configuration parameters + async fn approval_voting_params(&self, at: Hash) -> Result; } /// Default implementation of [`RuntimeApiSubsystemClient`] using the client. @@ -473,6 +477,11 @@ where runtime_api.submit_report_dispute_lost(at, dispute_proof, key_ownership_proof) } + /// Approval voting configuration parameters + async fn approval_voting_params(&self, at: Hash) -> Result { + self.client.runtime_api().approval_voting_params(at) + } + async fn staging_para_backing_state( &self, at: Hash, diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index 483256fe20f3..fb42b2c976ff 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -114,10 +114,11 @@ //! separated from the stable primitives. use crate::{ - vstaging, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, ValidatorSignature, + vstaging::{self, ApprovalVotingParams}, + BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, + CoreState, DisputeState, ExecutorParams, GroupRotationInfo, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidatorId, ValidatorIndex, ValidatorSignature, }; use parity_scale_codec::{Decode, Encode}; use polkadot_core_primitives as pcp; @@ -240,6 +241,9 @@ sp_api::decl_runtime_apis! { key_ownership_proof: vstaging::slashing::OpaqueKeyOwnershipProof, ) -> Option<()>; + /// Approval voting configuration parameters + #[api_version(99)] + fn approval_voting_params() -> ApprovalVotingParams; /***** Asynchronous backing *****/ /// Returns the state of parachain backing for a given para. diff --git a/polkadot/primitives/src/v5/mod.rs b/polkadot/primitives/src/v5/mod.rs index 59fb6c927b2d..1f5aa272fb2c 100644 --- a/polkadot/primitives/src/v5/mod.rs +++ b/polkadot/primitives/src/v5/mod.rs @@ -354,6 +354,13 @@ pub mod well_known_keys { /// Unique identifier for the Parachains Inherent pub const PARACHAINS_INHERENT_IDENTIFIER: InherentIdentifier = *b"parachn0"; +// /// TODO: Make this two a parachain host configuration. +// /// Maximum allowed candidates to be signed withing a signle approval votes. +// pub const MAX_APPROVAL_COALESCE_COUNT: u64 = 6; +// /// The maximum time we await for an approval to be coalesced with other approvals +// /// before we sign it and distribute to our peers +// pub const MAX_APPROVAL_COALESCE_WAIT_MILLIS: u64 = 500; + /// The key type ID for parachain assignment key. pub const ASSIGNMENT_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"asgn"); @@ -1115,6 +1122,26 @@ impl ApprovalVote { } } +/// A vote of approvalf for multiple candidates. +#[derive(Clone, RuntimeDebug)] +pub struct ApprovalVoteMultipleCandidates<'a>(pub &'a Vec); + +impl<'a> ApprovalVoteMultipleCandidates<'a> { + /// Yields the signing payload for this approval vote. + pub fn signing_payload(&self, session_index: SessionIndex) -> Vec { + const MAGIC: [u8; 4] = *b"APPR"; + // Make this backwards compatible with `ApprovalVote` so if we have just on candidate the signature + // will look the same. + // This gives us the nice benefit that old nodes can still check signatures when len is 1 and the + // new node can check the signature coming from old nodes. + if self.0.len() == 1 { + (MAGIC, self.0.first().expect("QED: we just checked"), session_index).encode() + } else { + (MAGIC, &self.0, session_index).encode() + } + } +} + /// Custom validity errors used in Polkadot while validating transactions. #[repr(u8)] pub enum ValidityError { @@ -1291,25 +1318,39 @@ pub enum DisputeStatement { impl DisputeStatement { /// Get the payload data for this type of dispute statement. - pub fn payload_data(&self, candidate_hash: CandidateHash, session: SessionIndex) -> Vec { - match *self { + pub fn payload_data( + &self, + candidate_hash: CandidateHash, + session: SessionIndex, + ) -> Result, ()> { + match self { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => - ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload(), + Ok(ExplicitDisputeStatement { valid: true, candidate_hash, session } + .signing_payload()), DisputeStatement::Valid(ValidDisputeStatementKind::BackingSeconded( inclusion_parent, - )) => CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { + )) => Ok(CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, - }), + parent_hash: *inclusion_parent, + })), DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) => - CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { + Ok(CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, - }), + parent_hash: *inclusion_parent, + })), DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) => - ApprovalVote(candidate_hash).signing_payload(session), + Ok(ApprovalVote(candidate_hash).signing_payload(session)), + DisputeStatement::Valid( + ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(candidate_hashes), + ) => + if candidate_hashes.contains(&candidate_hash) { + Ok(ApprovalVoteMultipleCandidates(candidate_hashes).signing_payload(session)) + } else { + Err(()) + }, DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) => - ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload(), + Ok(ExplicitDisputeStatement { valid: false, candidate_hash, session } + .signing_payload()), } } @@ -1321,7 +1362,7 @@ impl DisputeStatement { session: SessionIndex, validator_signature: &ValidatorSignature, ) -> Result<(), ()> { - let payload = self.payload_data(candidate_hash, session); + let payload = self.payload_data(candidate_hash, session)?; if validator_signature.verify(&payload[..], &validator_public) { Ok(()) @@ -1353,13 +1394,14 @@ impl DisputeStatement { Self::Valid(ValidDisputeStatementKind::BackingValid(_)) => true, Self::Valid(ValidDisputeStatementKind::Explicit) | Self::Valid(ValidDisputeStatementKind::ApprovalChecking) | + Self::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(_)) | Self::Invalid(_) => false, } } } /// Different kinds of statements of validity on a candidate. -#[derive(Encode, Decode, Copy, Clone, PartialEq, RuntimeDebug, TypeInfo)] +#[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)] pub enum ValidDisputeStatementKind { /// An explicit statement issued as part of a dispute. #[codec(index = 0)] @@ -1373,6 +1415,11 @@ pub enum ValidDisputeStatementKind { /// An approval vote from the approval checking phase. #[codec(index = 3)] ApprovalChecking, + /// An approval vote from the new version. + /// TODO: Fixme this probably means we can't create this version + /// untill all nodes have been updated to support it. + #[codec(index = 4)] + ApprovalCheckingMultipleCandidates(Vec), } /// Different kinds of statements of invalidity on a candidate. diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index ea341ee5b4fc..2b9d14fb199a 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -54,6 +54,26 @@ pub struct AsyncBackingParams { pub allowed_ancestry_len: u32, } +/// Approval voting configuration parameters +#[derive( + RuntimeDebug, + Copy, + Clone, + PartialEq, + Encode, + Decode, + TypeInfo, + serde::Serialize, + serde::Deserialize, +)] +pub struct ApprovalVotingParams { + /// The maximum number of candidates `approval-voting` can vote for with + /// a single signatures. + /// + /// Setting it to 1, means we send the approval as soon as we have it available. + pub max_approval_coalesce_count: u32, +} + /// Constraints on inbound HRMP channels. #[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] pub struct InboundHrmpLimitations { diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index 375b8f1f12b3..3f652b55ef9f 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -2,7 +2,7 @@ Reading the [section on the approval protocol](../../protocol-approval.md) will likely be necessary to understand the aims of this subsystem. -Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to indicate intent to check a candidate. Upon successfully checking, they broadcast an approval vote. If a validator doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being prevented from recovering or validating the block data and that more validators should self-select to check the candidate. This is known as a "no-show". +Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to indicate intent to check a candidate. Upon successfully checking, they don't immediately send the vote instead they queue the check for a short period of time `MAX_APPROVALS_COALESCE_TICKS` to give the opportunity of the validator to vote for more than one candidate. Once MAX_APPROVALS_COALESCE_TICKS have passed or at least `MAX_APPROVAL_COALESCE_COUNT` are ready they broadcast an approval vote for all candidates. If a validator doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being prevented from recovering or validating the block data and that more validators should self-select to check the candidate. This is known as a "no-show". The core of this subsystem is a Tick-based timer loop, where Ticks are 500ms. We also reason about time in terms of `DelayTranche`s, which measure the number of ticks elapsed since a block was produced. We track metadata for all un-finalized but included candidates. We compute our local assignments to check each candidate, as well as which `DelayTranche` those assignments may be minimally triggered at. As the same candidate may appear in more than one block, we must produce our potential assignments for each (Block, Candidate) pair. The timing loop is based on waiting for assignments to become no-shows or waiting to broadcast and begin our own assignment to check. @@ -94,6 +94,13 @@ struct BlockEntry { // this block. The block can be considered approved has all bits set to 1 approved_bitfield: Bitfield, children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + candidates_pending_signature: BTreeMap, + // Assignments we already distributed. A 1 bit means the candidate index for which + // we already have sent out an assignment. We need this to avoid distributing + // multiple core assignments more than once. + distributed_assignments: Bitfield, } // slot_duration * 2 + DelayTranche gives the number of delay tranches since the @@ -224,10 +231,10 @@ On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we che On receiving a `CheckAndImportApproval(indirect_approval_vote, response_channel)` message: * Fetch the `BlockEntry` from the indirect approval vote's `block_hash`. If none, return `ApprovalCheckResult::Bad`. - * Fetch the `CandidateEntry` from the indirect approval vote's `candidate_index`. If the block did not trigger inclusion of enough candidates, return `ApprovalCheckResult::Bad`. - * Construct a `SignedApprovalVote` using the candidate hash and check against the validator's approval key, based on the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. + * Fetch all `CandidateEntry` from the indirect approval vote's `candidate_indices`. If the block did not trigger inclusion of enough candidates, return `ApprovalCheckResult::Bad`. + * Construct a `SignedApprovalVote` using the candidates hashes and check against the validator's approval key, based on the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. * Send `ApprovalCheckResult::Accepted` - * [Import the checked approval vote](#import-checked-approval) + * [Import the checked approval vote](#import-checked-approval) for all candidates #### `ApprovalVotingMessage::ApprovedAncestor` @@ -295,10 +302,24 @@ On receiving an `ApprovedAncestor(Hash, BlockNumber, response_channel)`: #### Issue Approval Vote * Fetch the block entry and candidate entry. Ignore if `None` - we've probably just lost a race with finality. - * Construct a `SignedApprovalVote` with the validator index for the session. * [Import the checked approval vote](#import-checked-approval). It is "checked" as we've just issued the signature. - * Construct a `IndirectSignedApprovalVote` using the information about the vote. - * Dispatch `ApprovalDistributionMessage::DistributeApproval`. + * IF `MAX_APPROVAL_COALESCE_COUNT` candidates are in the waiting queue + * Construct a `SignedApprovalVote` with the validator index for the session and all candidate hashes in the waiting queue. + * Construct a `IndirectSignedApprovalVote` using the information about the vote. + * Dispatch `ApprovalDistributionMessage::DistributeApproval`. + * ELSE + * Queue the candidate in the `BlockEntry::candidates_pending_signature` + * Arm a per BlockEntry timer with latest tick we can send the vote. + +### Delayed vote distribution + * [Issue Approval Vote](#issue-approval-vote) arms once a per block timer if there are no requirements to send the vote immediately. + * When the timer wakes up it will either: + * IF there is a candidate in the queue past its sending tick: + * Construct a `SignedApprovalVote` with the validator index for the session and all candidate hashes in the waiting queue. + * Construct a `IndirectSignedApprovalVote` using the information about the vote. + * Dispatch `ApprovalDistributionMessage::DistributeApproval`. + * ELSE + * Re-arm the timer with latest tick we have the send a the vote. ### Determining Approval of Candidate diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index aa513c16292d..63345032cbb0 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -152,6 +152,12 @@ We strongly prefer if postponements come from tranches higher aka less important TODO: When? Is this optimal for the network? etc. +## Approval coalescing +To reduce the necessary network bandwidth and cpu time when a validator has more than one candidate to approve we are doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we delay the sending of it untill one of three things happen: +- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we are ready to vote for. +- `MAX_APPROVALS_COALESCE_TICKS` have passed since the we were ready to approve the candidate. +- We are already in the last third of the now-show period in order to avoid creating accidental no shows, which in turn my trigger other assignments. + ## On-chain verification We should verify approval on-chain to reward approval checkers. We therefore require the "no show" timeout to be longer than a relay chain slot so that we can witness "no shows" on-chain, which helps with this goal. The major challenge with an on-chain record of the off-chain process is adversarial block producers who may either censor votes or publish votes to the chain which cause other votes to be ignored and unrewarded (reward stealing). diff --git a/polkadot/runtime/kusama/src/lib.rs b/polkadot/runtime/kusama/src/lib.rs index e9e3fb2d2026..07bee8cfa853 100644 --- a/polkadot/runtime/kusama/src/lib.rs +++ b/polkadot/runtime/kusama/src/lib.rs @@ -23,12 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, - PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ auctions, claims, crowdloan, impl_runtime_weights, impls::DealWithFees, paras_registrar, @@ -1887,6 +1887,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 4921af5bedda..c2e5c917ff1a 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -634,7 +634,7 @@ impl BenchBuilder { } else { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) }; - let data = dispute_statement.payload_data(candidate_hash, session); + let data = dispute_statement.payload_data(candidate_hash, session).unwrap(); let statement_sig = validator_public.sign(&data).unwrap(); (dispute_statement, ValidatorIndex(validator_index), statement_sig) diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index accc01a2b180..47f4d1547368 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -24,8 +24,9 @@ use frame_system::pallet_prelude::*; use parity_scale_codec::{Decode, Encode}; use polkadot_parachain::primitives::{MAX_HORIZONTAL_MESSAGE_NUM, MAX_UPWARD_MESSAGE_NUM}; use primitives::{ - vstaging::AsyncBackingParams, Balance, ExecutorParams, SessionIndex, MAX_CODE_SIZE, - MAX_HEAD_DATA_SIZE, MAX_POV_SIZE, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + vstaging::{ApprovalVotingParams, AsyncBackingParams}, + Balance, ExecutorParams, SessionIndex, MAX_CODE_SIZE, MAX_HEAD_DATA_SIZE, MAX_POV_SIZE, + ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, }; use sp_runtime::{traits::Zero, Perbill}; use sp_std::prelude::*; @@ -242,6 +243,10 @@ pub struct HostConfiguration { /// /// This value should be greater than [`paras_availability_period`]. pub minimum_validation_upgrade_delay: BlockNumber, + + /// Params used by approval-voting + /// TODO: fixme this is not correctly migrated + pub approval_voting_params: ApprovalVotingParams, } impl> Default for HostConfiguration { @@ -287,6 +292,7 @@ impl> Default for HostConfiguration crate::hrmp::HRMP_MAX_INBOUND_CHANNELS_BOUND { return Err(MaxHrmpInboundChannelsExceeded) } - + // TODO: add consistency check for approval-voting-params Ok(()) } @@ -1150,6 +1156,22 @@ pub mod pallet { config.on_demand_ttl = new; }) } + + /// Set approval-voting-params. + #[pallet::call_index(52)] + #[pallet::weight(( + T::WeightInfo::set_config_with_executor_params(), + DispatchClass::Operational, + ))] + pub fn set_approval_voting_params( + origin: OriginFor, + new: ApprovalVotingParams, + ) -> DispatchResult { + ensure_root(origin)?; + Self::schedule_config_update(|config| { + config.approval_voting_params = new; + }) + } } #[pallet::hooks] diff --git a/polkadot/runtime/parachains/src/configuration/migration/v7.rs b/polkadot/runtime/parachains/src/configuration/migration/v7.rs index 113651381207..3624eb982323 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v7.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v7.rs @@ -240,6 +240,7 @@ pvf_voting_ttl : pre.pvf_voting_ttl, minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, async_backing_params : pre.async_backing_params, executor_params : pre.executor_params, + } }; diff --git a/polkadot/runtime/parachains/src/configuration/migration/v8.rs b/polkadot/runtime/parachains/src/configuration/migration/v8.rs index 7f7cc1cdefcd..d39f76100a18 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v8.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v8.rs @@ -23,7 +23,7 @@ use frame_support::{ weights::Weight, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::SessionIndex; +use primitives::{vstaging::ApprovalVotingParams, SessionIndex}; use sp_runtime::Perbill; use sp_std::vec::Vec; @@ -150,6 +150,9 @@ on_demand_base_fee : 10_000_000u128, on_demand_fee_variability : Perbill::from_percent(3), on_demand_target_queue_utilization : Perbill::from_percent(25), on_demand_ttl : 5u32.into(), +approval_voting_params : ApprovalVotingParams { + max_approval_coalesce_count: 1, + } } }; diff --git a/polkadot/runtime/parachains/src/configuration/tests.rs b/polkadot/runtime/parachains/src/configuration/tests.rs index 43c03067a9a7..72d1202db61f 100644 --- a/polkadot/runtime/parachains/src/configuration/tests.rs +++ b/polkadot/runtime/parachains/src/configuration/tests.rs @@ -312,6 +312,7 @@ fn setting_pending_config_members() { pvf_voting_ttl: 3, minimum_validation_upgrade_delay: 20, executor_params: Default::default(), + approval_voting_params: ApprovalVotingParams { max_approval_coalesce_count: 1 }, on_demand_queue_max_size: 10_000u32, on_demand_base_fee: 10_000_000u128, on_demand_fee_variability: Perbill::from_percent(3), diff --git a/polkadot/runtime/parachains/src/disputes.rs b/polkadot/runtime/parachains/src/disputes.rs index cf2e99e7359a..e0ff68f79063 100644 --- a/polkadot/runtime/parachains/src/disputes.rs +++ b/polkadot/runtime/parachains/src/disputes.rs @@ -25,11 +25,11 @@ use frame_system::pallet_prelude::*; use parity_scale_codec::{Decode, Encode}; use polkadot_runtime_metrics::get_current_time; use primitives::{ - byzantine_threshold, supermajority_threshold, ApprovalVote, CandidateHash, - CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CompactStatement, ConsensusLog, - DisputeState, DisputeStatement, DisputeStatementSet, ExplicitDisputeStatement, - InvalidDisputeStatementKind, MultiDisputeStatementSet, SessionIndex, SigningContext, - ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorSignature, + byzantine_threshold, supermajority_threshold, vstaging::ApprovalVoteMultipleCandidates, + ApprovalVote, CandidateHash, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, + CompactStatement, ConsensusLog, DisputeState, DisputeStatement, DisputeStatementSet, + ExplicitDisputeStatement, InvalidDisputeStatementKind, MultiDisputeStatementSet, SessionIndex, + SigningContext, ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorSignature, }; use scale_info::TypeInfo; use sp_runtime::{ @@ -1016,6 +1016,8 @@ impl Pallet { statement, signature, ) { + log::warn!("Failed to check dispute signature"); + importer.undo(undo); filter.remove_index(i); continue @@ -1261,21 +1263,29 @@ fn check_signature( statement: &DisputeStatement, validator_signature: &ValidatorSignature, ) -> Result<(), ()> { - let payload = match *statement { + let payload = match statement { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => ExplicitDisputeStatement { valid: true, candidate_hash, session }.signing_payload(), DisputeStatement::Valid(ValidDisputeStatementKind::BackingSeconded(inclusion_parent)) => CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, + parent_hash: *inclusion_parent, }), DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) => CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { session_index: session, - parent_hash: inclusion_parent, + parent_hash: *inclusion_parent, }), DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) => ApprovalVote(candidate_hash).signing_payload(session), + DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates( + candidates, + )) => + if candidates.contains(&candidate_hash) { + ApprovalVoteMultipleCandidates(candidates).signing_payload(session) + } else { + return Err(()) + }, DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) => ExplicitDisputeStatement { valid: false, candidate_hash, session }.signing_payload(), }; diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs b/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs index 5406428377d0..a9c433451808 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs @@ -16,6 +16,8 @@ //! Put implementations of functions from staging APIs here. +use primitives::vstaging::ApprovalVotingParams; + use crate::{configuration, dmp, hrmp, inclusion, initializer, paras, shared}; use frame_system::pallet_prelude::BlockNumberFor; use primitives::{ @@ -27,6 +29,11 @@ use primitives::{ }; use sp_std::prelude::*; +pub fn approval_voting_params() -> ApprovalVotingParams { + let config = >::config(); + config.approval_voting_params +} + /// Implementation for `StagingParaBackingState` function from the runtime API pub fn backing_state( para_id: ParaId, diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index c4458076cb3d..e7adbdc9be16 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -60,12 +60,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, - PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, }; use sp_core::OpaqueMetadata; use sp_mmr_primitives as mmr; @@ -1690,6 +1690,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 6894bd7bbf44..25cf8f21de69 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -23,11 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, claims, crowdloan, impl_runtime_weights, impls::ToAuthor, @@ -1714,6 +1715,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 9ae30c376010..717423e0b124 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -42,12 +42,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{CurrencyAdapter, FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + PvfCheckStatement, ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, + ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, crowdloan, elections::OnChainAccuracy, impl_runtime_weights, @@ -1561,6 +1561,7 @@ sp_api::impl_runtime_apis! { } } + #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/scripts/ci/gitlab/pipeline/build.yml b/polkadot/scripts/ci/gitlab/pipeline/build.yml index 845ac7970108..70d1b1fc5456 100644 --- a/polkadot/scripts/ci/gitlab/pipeline/build.yml +++ b/polkadot/scripts/ci/gitlab/pipeline/build.yml @@ -21,7 +21,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --verbose --bins + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime,network-protocol-staging --verbose --bins # pack artifacts - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name diff --git a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml index 1c8df44cbaa7..2384a5935661 100644 --- a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml +++ b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml @@ -64,6 +64,36 @@ zombienet-tests-parachains-pvf: tags: - zombienet-polkadot-integration-test +zombienet-tests-parachains-approval-coalescing: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0006-approval-voting-coalescing.zndsl" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + zombienet-tests-parachains-disputes: stage: zombienet image: "${ZOMBIENET_IMAGE}" diff --git a/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl b/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl index 46bb8bcdf72b..7859dd6d1156 100644 --- a/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl +++ b/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl @@ -32,6 +32,8 @@ alice: parachain 2005 block height is at least 10 within 300 seconds alice: parachain 2006 block height is at least 10 within 300 seconds alice: parachain 2007 block height is at least 10 within 300 seconds +alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds + # Check preparation time is under 10s. # Check all buckets <= 10. alice: reports histogram polkadot_pvf_preparation_time has at least 1 samples in buckets ["0.1", "0.5", "1", "2", "3", "10"] within 10 seconds diff --git a/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml b/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml index a0a87d60d4e3..5c649ceaf75f 100644 --- a/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml +++ b/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml @@ -5,6 +5,10 @@ timeout = 1000 max_validators_per_core = 5 needed_approvals = 8 +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config.approval_voting_params] + max_approval_coalesce_count = 5 + + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml new file mode 100644 index 000000000000..ec6d17dff4e7 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml @@ -0,0 +1,195 @@ +[settings] +timeout = 1000 + +[relaychain] +default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" +chain = "rococo-local" +chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] + needed_approvals = 5 + relay_vrf_modulo_samples = 3 + +[relaychain.genesis.runtime.runtime_genesis_config.configuration.config.approval_voting_params] + max_approval_coalesce_count = 6 + +[relaychain.default_resources] +limits = { memory = "4G", cpu = "2" } +requests = { memory = "2G", cpu = "1" } + + [[relaychain.nodes]] + name = "alice" + args = [ "--alice", "-lparachain=debug,runtime=debug,peerset=trace" ] + + [[relaychain.nodes]] + name = "bob" + args = [ "--bob", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "charlie" + args = [ "--charlie", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "dave" + args = [ "--dave", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "ferdie" + args = [ "--ferdie", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "eve" + args = [ "--eve", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "one" + args = [ "--one", "-lparachain=debug,runtime=debug" ] + + [[relaychain.nodes]] + name = "two" + args = [ "--two", "-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "nine" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "eleven" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twelve" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "thirteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "fourteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "fifteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "sixteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "seventeen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "eithteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "nineteen" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twenty" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twentyone" + args = ["-lparachain=debug,runtime=debug"] + + [[relaychain.nodes]] + name = "twentytwo" + args = ["-lparachain=debug,runtime=debug"] + +[[parachains]] +id = 2000 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=1" + + [parachains.collator] + name = "collator01" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=1", "--parachain-id=2000"] + +[[parachains]] +id = 2001 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=10" + + [parachains.collator] + name = "collator02" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2001", "--pvf-complexity=10"] + +[[parachains]] +id = 2002 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=100" + + [parachains.collator] + name = "collator03" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2002", "--pvf-complexity=100"] + +[[parachains]] +id = 2003 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=300" + + [parachains.collator] + name = "collator04" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--parachain-id=2003", "--pvf-complexity=300"] + +[[parachains]] +id = 2004 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator05" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--parachain-id=2004", "--pvf-complexity=300"] + +[[parachains]] +id = 2005 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=20000 --pvf-complexity=400" + + [parachains.collator] + name = "collator06" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=20000", "--pvf-complexity=400", "--parachain-id=2005"] + +[[parachains]] +id = 2006 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator07" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2006"] + +[[parachains]] +id = 2007 +addToGenesis = true +genesis_state_generator = "undying-collator export-genesis-state --pov-size=100000 --pvf-complexity=300" + + [parachains.collator] + name = "collator08" + image = "{{COL_IMAGE}}" + command = "undying-collator" + args = ["-lparachain=debug", "--pov-size=100000", "--pvf-complexity=300", "--parachain-id=2007"] + +[types.Header] +number = "u64" +parent_hash = "Hash" +post_state = "Hash" \ No newline at end of file diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl new file mode 100644 index 000000000000..272ac4fa2640 --- /dev/null +++ b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl @@ -0,0 +1,51 @@ +Description: Approval voting coalescing does not lag finality +Network: ./0006-approval-voting-coalescing.toml +Creds: config + +# Check authority status. +alice: reports node_roles is 4 +bob: reports node_roles is 4 +charlie: reports node_roles is 4 +dave: reports node_roles is 4 +eve: reports node_roles is 4 +ferdie: reports node_roles is 4 +one: reports node_roles is 4 +two: reports node_roles is 4 + +# Ensure parachains are registered. +alice: parachain 2000 is registered within 60 seconds +bob: parachain 2001 is registered within 60 seconds +charlie: parachain 2002 is registered within 60 seconds +dave: parachain 2003 is registered within 60 seconds +ferdie: parachain 2004 is registered within 60 seconds +eve: parachain 2005 is registered within 60 seconds +one: parachain 2006 is registered within 60 seconds +two: parachain 2007 is registered within 60 seconds + +# Ensure parachains made progress. +alice: parachain 2000 block height is at least 10 within 300 seconds +alice: parachain 2001 block height is at least 10 within 300 seconds +alice: parachain 2002 block height is at least 10 within 300 seconds +alice: parachain 2003 block height is at least 10 within 300 seconds +alice: parachain 2004 block height is at least 10 within 300 seconds +alice: parachain 2005 block height is at least 10 within 300 seconds +alice: parachain 2006 block height is at least 10 within 300 seconds +alice: parachain 2007 block height is at least 10 within 300 seconds + +alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +bob: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +charlie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +dave: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +eve: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +ferdie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +one: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds +two: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds + +alice: reports polkadot_parachain_approval_checking_finality_lag is 0 +bob: reports polkadot_parachain_approval_checking_finality_lag is 0 +charlie: reports polkadot_parachain_approval_checking_finality_lag is 0 +dave: reports polkadot_parachain_approval_checking_finality_lag is 0 +ferdie: reports polkadot_parachain_approval_checking_finality_lag is 0 +eve: reports polkadot_parachain_approval_checking_finality_lag is 0 +one: reports polkadot_parachain_approval_checking_finality_lag is 0 +two: reports polkadot_parachain_approval_checking_finality_lag is 0 \ No newline at end of file From 619fff2e6221a704669b45bbef17354626ddd88f Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 27 Aug 2023 14:05:00 +0300 Subject: [PATCH 04/89] Fix build warnings Signed-off-by: Alexandru Gheorghe --- polkadot/node/service/src/fake_runtime_api.rs | 12 ++++++------ polkadot/runtime/kusama/src/lib.rs | 12 ++++++------ polkadot/runtime/polkadot/src/lib.rs | 12 ++++++------ polkadot/runtime/rococo/src/lib.rs | 11 +++++------ polkadot/runtime/westend/src/lib.rs | 12 ++++++------ 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs index f2f2bdefae7a..a8ec22baf791 100644 --- a/polkadot/node/service/src/fake_runtime_api.rs +++ b/polkadot/node/service/src/fake_runtime_api.rs @@ -23,12 +23,12 @@ use beefy_primitives::ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefyS use grandpa_primitives::AuthorityId as GrandpaId; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use polkadot_primitives::{ - runtime_api, slashing, vstaging::ApprovalVotingParams, AccountId, AuthorityDiscoveryId, - Balance, Block, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Nonce, OccupiedCoreAssumption, - PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, + runtime_api, slashing, AccountId, AuthorityDiscoveryId, Balance, Block, BlockNumber, + CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, + DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Nonce, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, + ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, + ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_core::OpaqueMetadata; use sp_runtime::{ diff --git a/polkadot/runtime/kusama/src/lib.rs b/polkadot/runtime/kusama/src/lib.rs index 07bee8cfa853..56fdcec7165a 100644 --- a/polkadot/runtime/kusama/src/lib.rs +++ b/polkadot/runtime/kusama/src/lib.rs @@ -23,12 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, - CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, - ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, - ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, + slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, + OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, + PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ auctions, claims, crowdloan, impl_runtime_weights, impls::DealWithFees, paras_registrar, diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index e7adbdc9be16..b37ebaeff1a4 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -60,12 +60,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, - CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, - ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, - ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, LOWEST_PUBLIC_ID, PARACHAIN_KEY_TYPE_ID, + slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, + OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, + PARACHAIN_KEY_TYPE_ID, }; use sp_core::OpaqueMetadata; use sp_mmr_primitives as mmr; diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 25cf8f21de69..042caeee936a 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -23,12 +23,11 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, - CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, - ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, - ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, PARACHAIN_KEY_TYPE_ID, + slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, + OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, claims, crowdloan, impl_runtime_weights, impls::ToAuthor, diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 717423e0b124..a7b2782591b5 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -42,12 +42,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{CurrencyAdapter, FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, - CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, - ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, - PvfCheckStatement, ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, - ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, PARACHAIN_KEY_TYPE_ID, + slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, + CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, + OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, + SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, + ValidatorSignature, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, crowdloan, elections::OnChainAccuracy, impl_runtime_weights, From ed1d9d0c3091524694b41abf3eaee38bac248e84 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 09:43:52 +0300 Subject: [PATCH 05/89] ci: fix worker binaries could not be found Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 4 ++++ docker/malus_injected.Dockerfile | 2 +- docker/polkadot_injected_debug.Dockerfile | 6 ++++-- docker/polkadot_injected_release.Dockerfile | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 290ef8c8f72d..c46aa895a4eb 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -23,6 +23,8 @@ build-linux-stable: - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name - mv ./target/testnet/polkadot ./artifacts/. + - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. + - mv ./target/testnet/polkadot-execute-worker ./artifacts/. - pushd artifacts - sha256sum polkadot | tee polkadot.sha256 - shasum -c polkadot.sha256 @@ -68,6 +70,8 @@ build-malus: # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. + - mv ./target/testnet/polkadot-execute-worker ./artifacts/. + - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" diff --git a/docker/malus_injected.Dockerfile b/docker/malus_injected.Dockerfile index ecffd2c4f9b4..762a2e442c82 100644 --- a/docker/malus_injected.Dockerfile +++ b/docker/malus_injected.Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && \ # add adder-collator binary to docker image -COPY ./artifacts/malus /usr/local/bin +COPY ./artifacts/malus ./artifacts/polkadot-execute-worker ./artifacts/polkadot-prepare-worker /usr/local/bin USER nonroot diff --git a/docker/polkadot_injected_debug.Dockerfile b/docker/polkadot_injected_debug.Dockerfile index 3dd62f7ba56f..f7f764d335a2 100644 --- a/docker/polkadot_injected_debug.Dockerfile +++ b/docker/polkadot_injected_debug.Dockerfile @@ -32,13 +32,15 @@ RUN apt-get update && \ chown -R polkadot:polkadot /data && \ ln -s /data /polkadot/.local/share/polkadot -# add polkadot binary to docker image -COPY ./artifacts/polkadot /usr/local/bin +# add polkadot binaries to docker image +COPY ./artifacts/polkadot ./artifacts/polkadot-execute-worker ./artifacts/polkadot-prepare-worker /usr/local/bin USER polkadot # check if executable works in this container RUN /usr/local/bin/polkadot --version +RUN /usr/local/bin/polkadot-execute-worker --version +RUN /usr/local/bin/polkadot-prepare-worker --version EXPOSE 30333 9933 9944 VOLUME ["/polkadot"] diff --git a/docker/polkadot_injected_release.Dockerfile b/docker/polkadot_injected_release.Dockerfile index ba0a79e78187..87ae7ac27dc0 100644 --- a/docker/polkadot_injected_release.Dockerfile +++ b/docker/polkadot_injected_release.Dockerfile @@ -44,6 +44,8 @@ USER polkadot # check if executable works in this container RUN /usr/bin/polkadot --version +RUN /usr/local/bin/polkadot-execute-worker --version +RUN /usr/local/bin/polkadot-prepare-worker --version EXPOSE 30333 9933 9944 VOLUME ["/polkadot"] From 7d7b82c68918183bb549d1d7b759836b762a6304 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 10:20:33 +0300 Subject: [PATCH 06/89] Add missing bits Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 4 ++-- polkadot/Cargo.toml | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index c46aa895a4eb..422bea02efee 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker # pack artifacts - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name @@ -66,7 +66,7 @@ build-malus: - .run-immediately - .collect-artifacts script: - - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus + - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus --bin polkadot-prepare-worker --bin polkadot-execute-worker # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. diff --git a/polkadot/Cargo.toml b/polkadot/Cargo.toml index 561b49ab4204..925d6589dc0b 100644 --- a/polkadot/Cargo.toml +++ b/polkadot/Cargo.toml @@ -159,6 +159,16 @@ assets = [ "/usr/bin/", "755", ], + [ + "target/release/polkadot-prepare-worker", + "/usr/lib/polkadot/", + "755" + ], + [ + "target/release/polkadot-execute-worker", + "/usr/lib/polkadot/", + "755" + ], [ "scripts/packaging/polkadot.service", "/lib/systemd/system/", From 7bc13d311b273238cd6f4606f55e61ae5fbe9970 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 12:21:25 +0300 Subject: [PATCH 07/89] Build with network-protocol-staging Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 422bea02efee..7f5c86ff5f68 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker # pack artifacts - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name From 53f8556c501baaaebf6059366bcbece06294e4cd Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 11:46:22 +0300 Subject: [PATCH 08/89] Validate disconnect theory Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/approval-distribution/src/lib.rs | 6 +++--- .../network/statement-distribution/src/legacy_v1/mod.rs | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 3b8624e9166d..bdc2f859b909 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -583,9 +583,9 @@ impl State { NetworkBridgeEvent::PeerDisconnected(peer_id) => { gum::trace!(target: LOG_TARGET, ?peer_id, "Peer disconnected"); self.peer_views.remove(&peer_id); - self.blocks.iter_mut().for_each(|(_hash, entry)| { - entry.known_by.remove(&peer_id); - }) + // self.blocks.iter_mut().for_each(|(_hash, entry)| { + // entry.known_by.remove(&peer_id); + // }) }, NetworkBridgeEvent::NewGossipTopology(topology) => { self.handle_new_session_topology( diff --git a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs index b5895cb9f65b..938eb7fe64c0 100644 --- a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs +++ b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs @@ -1195,6 +1195,10 @@ async fn modify_reputation( peer: PeerId, rep: Rep, ) { + //TODO: Test teory in versi + if rep == COST_DUPLICATE_STATEMENT { + return + } reputation.modify(sender, peer, rep).await; } From 5e004e16e23525fbfe423b4ea15cd36d750e5646 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 28 Aug 2023 20:27:18 +0300 Subject: [PATCH 09/89] Log errors when banning peers Signed-off-by: Alexandru Gheorghe --- substrate/client/network/src/peer_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/network/src/peer_store.rs b/substrate/client/network/src/peer_store.rs index 2f3d4a1fd1a0..96bcd47cc12e 100644 --- a/substrate/client/network/src/peer_store.rs +++ b/substrate/client/network/src/peer_store.rs @@ -222,7 +222,7 @@ impl PeerStoreInner { if peer_info.reputation < BANNED_THRESHOLD { self.protocols.iter().for_each(|handle| handle.disconnect_peer(peer_id)); - log::trace!( + log::error!( target: LOG_TARGET, "Report {}: {:+} to {}. Reason: {}. Banned, disconnecting.", peer_id, From 9850b2ff909b506f81178776bd277125306cb109 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 28 Aug 2023 23:47:09 +0300 Subject: [PATCH 10/89] fix zombienet test Signed-off-by: Andrei Sandu --- .gitlab/pipeline/zombienet/polkadot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 82dd13cd290c..7bc23db3259a 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -79,6 +79,14 @@ zombienet-polkadot-functional-0004-beefy-and-mmr: --local-dir="${LOCAL_DIR}/functional" --test="0003-beefy-and-mmr.zndsl" +zombienet-polkadot-functional-0005-max-tranche0: + extends: + - .zombienet-polkadot-common + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh + --local-dir="${LOCAL_DIR}/functional" + --test="0005-parachains-max-tranche0.zndsl" + zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: - .zombienet-polkadot-common From 46cfaf1e55e9e27452e1c9e805d29684eae9bf72 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 29 Aug 2023 19:06:20 +0300 Subject: [PATCH 11/89] cargo lock Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 0ccf5aa0ba5e..a586275ba332 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11952,6 +11952,7 @@ dependencies = [ "itertools 0.10.5", "kvdb", "kvdb-memorydb", + "log", "merlin 2.0.1", "parity-scale-codec", "parking_lot 0.12.1", From 47beabdf0675289a1a3e4cbb3843fed0937106b8 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 30 Aug 2023 18:07:12 +0300 Subject: [PATCH 12/89] superfluous Signed-off-by: Andrei Sandu --- .../approval-voting/src/approval_db/v2/migration_helpers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs index 3d1f28ee1937..a70d766fc8c9 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . //! Approval DB migration helpers. -use super::{StoredBlockRange, *}; +use super::*; use crate::backend::Backend; use polkadot_node_primitives::approval::v1::{ AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, From 3d3e37cf4810090c4a9f43b752ec4a9f4b000694 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 30 Aug 2023 13:27:04 +0300 Subject: [PATCH 13/89] Separate approval Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 69 +- .../network/approval-distribution/src/lib.rs | 593 ++++++++---------- .../approval-distribution/src/metrics.rs | 12 - .../approval-distribution/src/tests.rs | 474 +++++++++++++- 4 files changed, 786 insertions(+), 362 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 69c5603624d3..e8a19cd2bc2f 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -837,7 +837,7 @@ impl State { #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn get_approval_voting_params_or_default( &mut self, - _ctx: &mut Context, + ctx: &mut Context, block_hash: Hash, ) -> ApprovalVotingParams { if let Some(params) = self @@ -847,30 +847,29 @@ impl State { { *params } else { - // let (s_tx, s_rx) = oneshot::channel(); - - // ctx.send_message(RuntimeApiMessage::Request( - // block_hash, - // RuntimeApiRequest::ApprovalVotingParams(s_tx), - // )) - // .await; - - // match s_rx.await { - // Ok(Ok(params)) => { - // self.approval_voting_params_cache - // .as_mut() - // .map(|cache| cache.put(block_hash, params)); - // params - // }, - // _ => { - // gum::error!( - // target: LOG_TARGET, - // "Could not request approval voting params from runtime using defaults" - // ); - // TODO: Uncomment this after versi test - ApprovalVotingParams { max_approval_coalesce_count: 6 } - // }, - // } + let (s_tx, s_rx) = oneshot::channel(); + + ctx.send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::ApprovalVotingParams(s_tx), + )) + .await; + + match s_rx.await { + Ok(Ok(params)) => { + self.approval_voting_params_cache + .as_mut() + .map(|cache| cache.put(block_hash, params)); + params + }, + _ => { + gum::error!( + target: LOG_TARGET, + "Could not request approval voting params from runtime using defaults" + ); + ApprovalVotingParams { max_approval_coalesce_count: 6 } + }, + } } } } @@ -3276,7 +3275,12 @@ async fn issue_approval( .defer_candidate_signature( candidate_index as _, candidate_hash, - compute_delayed_approval_sending_tick(state, &block_entry, session_info), + compute_delayed_approval_sending_tick( + state, + &block_entry, + &candidate_entry, + session_info, + ), ) .is_some() { @@ -3532,9 +3536,17 @@ fn issue_local_invalid_statement( fn compute_delayed_approval_sending_tick( state: &State, block_entry: &BlockEntry, + candidate_entry: &CandidateEntry, session_info: &SessionInfo, ) -> Tick { let current_block_tick = slot_number_to_tick(state.slot_duration_millis, block_entry.slot()); + let assignment_tranche = candidate_entry + .approval_entry(&block_entry.block_hash()) + .and_then(|approval_entry| approval_entry.our_assignment()) + .map(|our_assignment| our_assignment.tranche()) + .unwrap(); + + let assignment_triggered_tick = current_block_tick + assignment_tranche as Tick; let no_show_duration_ticks = slot_number_to_tick( state.slot_duration_millis, @@ -3545,9 +3557,8 @@ fn compute_delayed_approval_sending_tick( min( tick_now + MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick, // We don't want to accidentally cause, no-shows so if we are past - // the 2 thirds of the no show time, force the sending of the + // the seconnd half of the no show time, force the sending of the // approval immediately. - // TODO: TBD the right value here, so that we don't accidentally create no-shows. - current_block_tick + (no_show_duration_ticks * 2) / 3, + assignment_triggered_tick + no_show_duration_ticks / 2, ) } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index bdc2f859b909..8d28a54dcd89 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -33,9 +33,7 @@ use polkadot_node_network_protocol::{ Versioned, View, }; use polkadot_node_primitives::approval::{ - v1::{ - AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert, IndirectSignedApprovalVote, - }, + v1::{AssignmentCertKind, BlockApprovalMeta, IndirectAssignmentCert}, v2::{ AsBitIndex, AssignmentCertKindV2, CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2, @@ -121,9 +119,9 @@ struct ApprovalEntry { // The assignment certificate. assignment: IndirectAssignmentCertV2, // The candidates claimed by the certificate. A mapping between bit index and candidate index. - candidates: CandidateBitfield, + assignment_claimed_candidates: CandidateBitfield, // The approval signatures for each `CandidateIndex` claimed by the assignment certificate. - approvals: HashMap, + approvals: HashMap, // The validator index of the assignment signer. validator_index: ValidatorIndex, // Information required for gossiping to other peers using the grid topology. @@ -136,6 +134,8 @@ enum ApprovalEntryError { CandidateIndexOutOfBounds, InvalidCandidateIndex, DuplicateApproval, + CouldNotFindAssignment, + AssignmentsFollowedDifferentPaths(RequiredRouting, RequiredRouting), } impl ApprovalEntry { @@ -148,7 +148,7 @@ impl ApprovalEntry { validator_index: assignment.validator, assignment, approvals: HashMap::with_capacity(candidates.len()), - candidates, + assignment_claimed_candidates: candidates, routing_info, } } @@ -156,23 +156,15 @@ impl ApprovalEntry { // Create a `MessageSubject` to reference the assignment. pub fn create_assignment_knowledge(&self, block_hash: Hash) -> (MessageSubject, MessageKind) { ( - MessageSubject(block_hash, self.candidates.clone(), self.validator_index), + MessageSubject( + block_hash, + self.assignment_claimed_candidates.clone(), + self.validator_index, + ), MessageKind::Assignment, ) } - // Create a `MessageSubject` to reference the approval. - pub fn create_approval_knowledge( - &self, - block_hash: Hash, - candidate_index: CandidateIndex, - ) -> (MessageSubject, MessageKind) { - ( - MessageSubject(block_hash, candidate_index.into(), self.validator_index), - MessageKind::Approval, - ) - } - // Updates routing information and returns the previous information if any. pub fn routing_info_mut(&mut self) -> &mut ApprovalRouting { &mut self.routing_info @@ -193,7 +185,6 @@ impl ApprovalEntry { pub fn note_approval( &mut self, approval: IndirectSignedApprovalVoteV2, - candidate_index: CandidateIndex, ) -> Result<(), ApprovalEntryError> { // First do some sanity checks: // - check validator index matches @@ -203,24 +194,28 @@ impl ApprovalEntry { return Err(ApprovalEntryError::InvalidValidatorIndex) } - if self.candidates.len() <= candidate_index as usize { - return Err(ApprovalEntryError::CandidateIndexOutOfBounds) - } - if !self.candidates.bit_at((candidate_index).as_bit_index()) { + // We need at least one of the candidates in the approval to be in this assignment + if !approval + .candidate_indices + .iter_ones() + .fold(false, |accumulator, candidate_index| { + self.assignment_claimed_candidates.bit_at((candidate_index).as_bit_index()) || + accumulator + }) { return Err(ApprovalEntryError::InvalidCandidateIndex) } - if self.approvals.contains_key(&candidate_index) { + if self.approvals.contains_key(&approval.candidate_indices) { return Err(ApprovalEntryError::DuplicateApproval) } - self.approvals.insert(candidate_index, approval.clone()); + self.approvals.insert(approval.candidate_indices.clone(), approval.clone()); Ok(()) } // Get the assignment certiticate and claimed candidates. pub fn assignment(&self) -> (IndirectAssignmentCertV2, CandidateBitfield) { - (self.assignment.clone(), self.candidates.clone()) + (self.assignment.clone(), self.assignment_claimed_candidates.clone()) } // Get all approvals for all candidates claimed by the assignment. @@ -228,14 +223,6 @@ impl ApprovalEntry { self.approvals.values().cloned().collect::>() } - // Get the approval for a specific candidate index. - pub fn approval( - &self, - candidate_index: CandidateIndex, - ) -> Option { - self.approvals.get(&candidate_index).cloned() - } - // Get validator index. pub fn validator_index(&self) -> ValidatorIndex { self.validator_index @@ -433,6 +420,51 @@ impl PeerKnowledge { fn contains(&self, message: &MessageSubject, kind: MessageKind) -> bool { self.sent.contains(message, kind) || self.received.contains(message, kind) } + + // Tells if all assignments for a given approval are included in the knowledge of the peer + fn contains_assignments_for_approval(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { + Self::generate_assignments_keys(&approval).iter().fold( + true, + |accumulator, assignment_key| { + accumulator && self.contains(&assignment_key.0, assignment_key.1) + }, + ) + } + + // Generate the knowledge keys for querying if all assignments of an approval are known + // by this peer. + fn generate_assignments_keys( + approval: &IndirectSignedApprovalVoteV2, + ) -> Vec<(MessageSubject, MessageKind)> { + approval + .candidate_indices + .iter_ones() + .map(|candidate_index| { + ( + MessageSubject( + approval.block_hash, + (candidate_index as CandidateIndex).into(), + approval.validator, + ), + MessageKind::Assignment, + ) + }) + .collect_vec() + } + + // Generate the knowledge keys for querying if an approval is known by peer. + fn generate_approval_key( + approval: &IndirectSignedApprovalVoteV2, + ) -> (MessageSubject, MessageKind) { + ( + MessageSubject( + approval.block_hash, + approval.candidate_indices.clone(), + approval.validator, + ), + MessageKind::Approval, + ) + } } /// Information about blocks in our current view as well as whether peers know of them. @@ -465,13 +497,13 @@ impl BlockEntry { // First map one entry per candidate to the same key we will use in `approval_entries`. // Key is (Validator_index, CandidateBitfield) that links the `ApprovalEntry` to the (K,V) // entry in `candidate_entry.messages`. - for claimed_candidate_index in entry.candidates.iter_ones() { + for claimed_candidate_index in entry.assignment_claimed_candidates.iter_ones() { match self.candidates.get_mut(claimed_candidate_index) { Some(candidate_entry) => { candidate_entry - .messages + .assignments .entry(entry.validator_index()) - .or_insert(entry.candidates.clone()); + .or_insert(entry.assignment_claimed_candidates.clone()); }, None => { // This should never happen, but if it happens, it means the subsystem is @@ -487,50 +519,90 @@ impl BlockEntry { } self.approval_entries - .entry((entry.validator_index, entry.candidates.clone())) + .entry((entry.validator_index, entry.assignment_claimed_candidates.clone())) .or_insert(entry) } - // Returns a mutable reference of `ApprovalEntry` for `candidate_index` from validator - // `validator_index`. - pub fn approval_entry( + // Tels if all candidate_indices are valid candidates + pub fn contains_candidates(&self, candidate_indices: &CandidateBitfield) -> bool { + candidate_indices.iter_ones().fold(true, |accumulator, candidate_index| { + self.candidates.get(candidate_index as usize).is_some() && accumulator + }) + } + // Saves the given approval in all ApprovalEntries that contain an assignment for any of the + // candidates in the approval. + // + // Returns the required routing needed for this approval. + pub fn note_approval( &mut self, - candidate_index: CandidateIndex, - validator_index: ValidatorIndex, - ) -> Option<&mut ApprovalEntry> { - self.candidates - .get(candidate_index as usize) - .map_or(None, |candidate_entry| candidate_entry.messages.get(&validator_index)) - .map_or(None, |candidate_indices| { - self.approval_entries.get_mut(&(validator_index, candidate_indices.clone())) + approval: IndirectSignedApprovalVoteV2, + ) -> Result { + let mut required_routing = None; + + if self.candidates.len() < approval.candidate_indices.len() as usize { + return Err(ApprovalEntryError::CandidateIndexOutOfBounds) + } + + // First determine all assignments bitfields that might be covered by this approval + let covered_assignments_bitfields: HashSet = approval + .candidate_indices + .iter_ones() + .filter_map(|candidate_index| { + self.candidates.get_mut(candidate_index).map_or(None, |candidate_entry| { + candidate_entry.assignments.get(&approval.validator).cloned() + }) }) - } + .collect(); - // Get all approval entries for a given candidate. - pub fn approval_entries(&self, candidate_index: CandidateIndex) -> Vec<&ApprovalEntry> { - // Get the keys for fetching `ApprovalEntry` from `self.approval_entries`, - let approval_entry_keys = self - .candidates - .get(candidate_index as usize) - .map(|candidate_entry| &candidate_entry.messages); - - if let Some(approval_entry_keys) = approval_entry_keys { - // Ensure no duplicates. - let approval_entry_keys = approval_entry_keys.iter().unique().collect::>(); - - let mut entries = Vec::new(); - for (validator_index, candidate_indices) in approval_entry_keys { - if let Some(entry) = - self.approval_entries.get(&(*validator_index, candidate_indices.clone())) - { - entries.push(entry); + // Mark the vote in all approval entries + for assignment_bitfield in covered_assignments_bitfields { + if let Some(approval_entry) = + self.approval_entries.get_mut(&(approval.validator, assignment_bitfield)) + { + approval_entry.note_approval(approval.clone())?; + + if let Some(required_routing) = required_routing { + if required_routing != approval_entry.routing_info().required_routing { + // This shouldn't happen since the required routing is computed based on the + // validator_index, so two assignments from the same validators will have + // the same required routing. + return Err(ApprovalEntryError::AssignmentsFollowedDifferentPaths( + required_routing, + approval_entry.routing_info().required_routing, + )) + } + } else { + required_routing = Some(approval_entry.routing_info().required_routing) } } - entries + } + + if let Some(required_routing) = required_routing { + Ok(required_routing) } else { - vec![] + Err(ApprovalEntryError::CouldNotFindAssignment) } } + + pub fn approval_votes( + &self, + candidate_index: CandidateIndex, + ) -> Vec { + let result: Option> = + self.candidates.get(candidate_index as usize).map(|candidate_entry| { + candidate_entry + .assignments + .iter() + .filter_map(|(validator, assignment_bitfield)| { + self.approval_entries.get(&(*validator, assignment_bitfield.clone())) + }) + .map(|approval_entry| approval_entry.approvals.clone().into_iter()) + .flatten() + .collect() + }); + + result.map(|result| result.into_values().collect_vec()).unwrap_or_default() + } } // Information about candidates in the context of a particular block they are included in. @@ -540,7 +612,7 @@ impl BlockEntry { struct CandidateEntry { // The value represents part of the lookup key in `approval_entries` to fetch the assignment // and existing votes. - messages: HashMap, + assignments: HashMap, } #[derive(Debug, Clone, PartialEq)] @@ -583,9 +655,9 @@ impl State { NetworkBridgeEvent::PeerDisconnected(peer_id) => { gum::trace!(target: LOG_TARGET, ?peer_id, "Peer disconnected"); self.peer_views.remove(&peer_id); - // self.blocks.iter_mut().for_each(|(_hash, entry)| { - // entry.known_by.remove(&peer_id); - // }) + self.blocks.iter_mut().for_each(|(_hash, entry)| { + entry.known_by.remove(&peer_id); + }) }, NetworkBridgeEvent::NewGossipTopology(topology) => { self.handle_new_session_topology( @@ -833,6 +905,7 @@ impl State { } } + // Entry point for processing an approval coming from a peer. async fn process_incoming_approvals( &mut self, ctx: &mut Context, @@ -1377,17 +1450,19 @@ impl State { } } + // Checks if an approval can be processed. + // Returns true if we can continue with processing the approval and false otherwise. async fn check_approval_can_be_processed( ctx: &mut Context, - message_subjects: &Vec, - message_kind: MessageKind, + assignments_knowledge_key: &Vec<(MessageSubject, MessageKind)>, + approval_knowledge_key: &(MessageSubject, MessageKind), entry: &mut BlockEntry, reputation: &mut ReputationAggregator, peer_id: PeerId, metrics: &Metrics, ) -> bool { - for message_subject in message_subjects { - if !entry.knowledge.contains(&message_subject, MessageKind::Assignment) { + for message_subject in assignments_knowledge_key { + if !entry.knowledge.contains(&message_subject.0, message_subject.1) { gum::trace!( target: LOG_TARGET, ?peer_id, @@ -1398,66 +1473,63 @@ impl State { metrics.on_approval_unknown_assignment(); return false } + } - // check if our knowledge of the peer already contains this approval - match entry.known_by.entry(peer_id) { - hash_map::Entry::Occupied(mut knowledge) => { - let peer_knowledge = knowledge.get_mut(); - if peer_knowledge.contains(&message_subject, message_kind) { - if !peer_knowledge.received.insert(message_subject.clone(), message_kind) { - gum::trace!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Duplicate approval", - ); + // check if our knowledge of the peer already contains this approval + match entry.known_by.entry(peer_id) { + hash_map::Entry::Occupied(mut knowledge) => { + let peer_knowledge = knowledge.get_mut(); + if peer_knowledge.contains(&approval_knowledge_key.0, approval_knowledge_key.1) { + if !peer_knowledge + .received + .insert(approval_knowledge_key.0.clone(), approval_knowledge_key.1) + { + gum::trace!( + target: LOG_TARGET, + ?peer_id, + ?approval_knowledge_key, + "Duplicate approval", + ); - modify_reputation( - reputation, - ctx.sender(), - peer_id, - COST_DUPLICATE_MESSAGE, - ) - .await; - metrics.on_approval_duplicate(); - } - return false - } - }, - hash_map::Entry::Vacant(_) => { - gum::debug!( - target: LOG_TARGET, - ?peer_id, - ?message_subject, - "Approval from a peer is out of view", - ); - modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE) + modify_reputation( + reputation, + ctx.sender(), + peer_id, + COST_DUPLICATE_MESSAGE, + ) .await; - metrics.on_approval_out_of_view(); - }, - } - } - - let good_known_approval = - message_subjects.iter().fold(false, |accumulator, message_subject| { - // if the approval is known to be valid, reward the peer - if entry.knowledge.contains(&message_subject, message_kind) { - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); + metrics.on_approval_duplicate(); } - // We already processed this approval no need to continue. - true - } else { - accumulator + return false } - }); - if good_known_approval { - gum::trace!(target: LOG_TARGET, ?peer_id, ?message_subjects, "Known approval"); + }, + hash_map::Entry::Vacant(_) => { + gum::debug!( + target: LOG_TARGET, + ?peer_id, + ?approval_knowledge_key, + "Approval from a peer is out of view", + ); + modify_reputation(reputation, ctx.sender(), peer_id, COST_UNEXPECTED_MESSAGE).await; + metrics.on_approval_out_of_view(); + }, + } + + if entry.knowledge.contains(&approval_knowledge_key.0, approval_knowledge_key.1) { + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge + .received + .insert(approval_knowledge_key.0.clone(), approval_knowledge_key.1); + } + + // We already processed this approval no need to continue. + gum::trace!(target: LOG_TARGET, ?peer_id, ?approval_knowledge_key, "Known approval"); metrics.on_approval_good_known(); modify_reputation(reputation, ctx.sender(), peer_id, BENEFIT_VALID_MESSAGE).await; + false + } else { + true } - - !good_known_approval } async fn import_and_circulate_approval( @@ -1486,19 +1558,7 @@ impl State { let validator_index = vote.validator; let candidate_indices = &vote.candidate_indices; let entry = match self.blocks.get_mut(&block_hash) { - Some(entry) - if vote.candidate_indices.iter_ones().fold(true, |result, candidate_index| { - let approval_entry_exists = entry.candidates.get(candidate_index as usize).is_some(); - if !approval_entry_exists { - gum::debug!( - target: LOG_TARGET, ?block_hash, ?candidate_index, validator_index = ?vote.validator, candidate_indices = ?vote.candidate_indices, - peer_id = ?source.peer_id(), "Received approval before assignment" - ); - metrics.on_approval_entry_not_found(); - } - approval_entry_exists && result - }) => - entry, + Some(entry) if entry.contains_candidates(&vote.candidate_indices) => entry, _ => { if let Some(peer_id) = source.peer_id() { if !self.recent_outdated_blocks.is_recent_outdated(&block_hash) { @@ -1518,7 +1578,6 @@ impl State { ) .await; metrics.on_approval_invalid_block(); - } else { metrics.on_approval_recent_outdated(); } @@ -1528,23 +1587,14 @@ impl State { }; // compute metadata on the assignment. - let message_subjects = candidate_indices - .iter_ones() - .map(|candidate_index| { - MessageSubject( - block_hash, - (candidate_index as CandidateIndex).into(), - validator_index, - ) - }) - .collect_vec(); - let message_kind = MessageKind::Approval; + let assignments_knowledge_keys = PeerKnowledge::generate_assignments_keys(&vote); + let approval_knwowledge_key = PeerKnowledge::generate_approval_key(&vote); if let Some(peer_id) = source.peer_id() { if !Self::check_approval_can_be_processed( ctx, - &message_subjects, - message_kind, + &assignments_knowledge_keys, + &approval_knwowledge_key, entry, &mut self.reputation, peer_id, @@ -1574,6 +1624,7 @@ impl State { target: LOG_TARGET, ?peer_id, ?result, + ?vote, "Checked approval", ); match result { @@ -1586,11 +1637,13 @@ impl State { ) .await; - for message_subject in &message_subjects { - entry.knowledge.insert(message_subject.clone(), message_kind); - if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { - peer_knowledge.received.insert(message_subject.clone(), message_kind); - } + entry + .knowledge + .insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); + if let Some(peer_knowledge) = entry.known_by.get_mut(&peer_id) { + peer_knowledge + .received + .insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } }, ApprovalCheckResult::Bad(error) => { @@ -1612,11 +1665,10 @@ impl State { }, } } else { - let all_approvals_imported_already = - message_subjects.iter().fold(true, |result, message_subject| { - !entry.knowledge.insert(message_subject.clone(), message_kind) && result - }); - if all_approvals_imported_already { + if !entry + .knowledge + .insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1) + { // if we already imported all approvals, there is no need to distribute it again gum::warn!( target: LOG_TARGET, @@ -1631,45 +1683,21 @@ impl State { } } - let mut required_routing = RequiredRouting::None; - - for candidate_index in candidate_indices.iter_ones() { - // The entry is created when assignment is imported, so we assume this exists. - let approval_entry = entry.approval_entry(candidate_index as _, validator_index); - if approval_entry.is_none() { - let peer_id = source.peer_id(); - // This indicates a bug in approval-distribution, since we check the knowledge at - // the begining of the function. - gum::warn!( - target: LOG_TARGET, - ?peer_id, - "Unknown approval assignment", - ); - // No rep change as this is caused by an issue - metrics.on_approval_unexpected(); - return - } - - let approval_entry = approval_entry.expect("Just checked above; qed"); - - if let Err(err) = approval_entry.note_approval(vote.clone(), candidate_index as _) { - // this would indicate a bug in approval-voting: - // - validator index mismatch - // - candidate index mismatch - // - duplicate approval + let required_routing = match entry.note_approval(vote.clone()) { + Ok(required_routing) => required_routing, + Err(err) => { gum::warn!( target: LOG_TARGET, hash = ?block_hash, - ?candidate_index, - ?validator_index, + validator_index = ?vote.validator, + candidate_bitfield = ?vote.candidate_indices, ?err, "Possible bug: Vote import failed", ); metrics.on_approval_bug(); return - } - required_routing = approval_entry.routing_info().required_routing; - } + }, + }; // Invariant: to our knowledge, none of the peers except for the `source` know about the // approval. @@ -1680,7 +1708,6 @@ impl State { let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); - let message_subjects_clone = message_subjects.clone(); let peer_filter = move |peer, knowledge: &PeerKnowledge| { if Some(peer) == source_peer.as_ref() { return false @@ -1698,8 +1725,8 @@ impl State { let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); in_topology || - message_subjects_clone.iter().fold(true, |result, message_subject| { - result && knowledge.sent.contains(message_subject, MessageKind::Assignment) + assignments_knowledge_keys.iter().fold(true, |result, message_subject| { + result && knowledge.sent.contains(&message_subject.0, message_subject.1) }) }; @@ -1714,9 +1741,7 @@ impl State { for peer in peers.iter() { // we already filtered peers above, so this should always be Some if let Some(entry) = entry.known_by.get_mut(&peer.0) { - for message_subject in &message_subjects { - entry.sent.insert(message_subject.clone(), message_kind); - } + entry.sent.insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } } @@ -1730,43 +1755,7 @@ impl State { "Sending an approval to peers", ); - let v1_peers = filter_by_peer_version(&peers, ValidationVersion::V1.into()); - let v2_peers = filter_by_peer_version(&peers, ValidationVersion::VStaging.into()); - - if !v1_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v1_peers, - Versioned::V1(protocol_v1::ValidationProtocol::ApprovalDistribution( - protocol_v1::ApprovalDistributionMessage::Approvals( - approvals - .clone() - .into_iter() - .filter(|approval| approval.candidate_indices.count_ones() == 1) - .map(|approval| { - approval - .try_into() - .expect("We checked len() == 1 so it should not fail; qed") - }) - .collect::>(), - ), - )), - )) - .await; - metrics.on_approval_sent_v1(); - } - - if !v2_peers.is_empty() { - ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v2_peers, - Versioned::VStaging( - protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals), - ), - ), - )) - .await; - metrics.on_approval_sent_v2(); - } + send_approvals_batched(ctx.sender(), approvals, &peers).await; } } @@ -1798,9 +1787,8 @@ impl State { }; let sigs = block_entry - .approval_entries(index) + .approval_votes(index) .into_iter() - .filter_map(|approval_entry| approval_entry.approval(index)) .map(|approval| { ( approval.validator, @@ -1837,7 +1825,7 @@ impl State { let _timer = metrics.time_unify_with_peer(); let mut assignments_to_send = Vec::new(); - let mut approvals_to_send = HashMap::new(); + let mut approvals_to_send = Vec::new(); let view_finalized_number = view.finalized_number; for head in view.into_iter() { @@ -1860,6 +1848,7 @@ impl State { let peer_knowledge = entry.known_by.entry(peer_id).or_default(); let topology = topologies.get_topology(entry.session); + let mut approvals_to_send_this_block = HashMap::new(); // We want to iterate the `approval_entries` of the block entry as these contain all // assignments that also link all approval votes. for approval_entry in entry.approval_entries.values_mut() { @@ -1908,55 +1897,27 @@ impl State { // Filter approval votes. for approval_message in approval_messages { - let (should_forward_approval, candidates_covered_by_approvals) = - approval_message.candidate_indices.iter_ones().fold( - (true, Vec::new()), - |(should_forward_approval, mut new_covered_approvals), - approval_candidate_index| { - let (message_subject, message_kind) = approval_entry - .create_approval_knowledge( - block, - approval_candidate_index as _, - ); - // The assignments for all candidates signed in the approval - // should already have been sent to the peer, otherwise we can't - // send our approval and risk breaking our reputation. - let should_forward_approval = should_forward_approval && - peer_knowledge - .contains(&message_subject, MessageKind::Assignment); - if !peer_knowledge.contains(&message_subject, message_kind) { - new_covered_approvals.push((message_subject, message_kind)) - } - - (should_forward_approval, new_covered_approvals) - }, - ); + let approval_knowledge = + PeerKnowledge::generate_approval_key(&approval_message); - if should_forward_approval { - if !approvals_to_send.contains_key(&( - approval_message.block_hash, - approval_message.validator, - approval_message.candidate_indices.clone(), - )) { - approvals_to_send.insert( - ( - approval_message.block_hash, - approval_message.validator, - approval_message.candidate_indices.clone(), - ), - approval_message, - ); + if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) { + if !approvals_to_send_this_block.contains_key(&approval_knowledge.0) { + approvals_to_send_this_block + .insert(approval_knowledge.0.clone(), approval_message); } - - candidates_covered_by_approvals.into_iter().for_each( - |(approval_knowledge, message_kind)| { - peer_knowledge.sent.insert(approval_knowledge, message_kind); - }, - ); } } } + // An approval can span multiple assignments/ApprovalEntries, so after we processed + // all of the assignments decide which of the approvals we can safely send, because + // all of assignments are already sent or about to be sent. + for (approval_key, approval) in approvals_to_send_this_block { + if peer_knowledge.contains_assignments_for_approval(&approval) { + approvals_to_send.push(approval); + peer_knowledge.sent.insert(approval_key, MessageKind::Approval); + } + } block = entry.parent_hash; } } @@ -1987,12 +1948,8 @@ impl State { "Sending approvals to unified peer", ); - send_approvals_batched( - sender, - approvals_to_send.into_values().collect_vec(), - &vec![(peer_id, protocol_version)], - ) - .await; + send_approvals_batched(sender, approvals_to_send, &vec![(peer_id, protocol_version)]) + .await; } } @@ -2223,6 +2180,8 @@ async fn adjust_required_routing_and_propagate continue, }; + let mut approvals_to_send_for_this_block = HashMap::new(); + // We just need to iterate the `approval_entries` of the block entry as these contain all // assignments that also link all approval votes. for approval_entry in block_entry.approval_entries.values_mut() { @@ -2259,32 +2218,30 @@ async fn adjust_required_routing_and_propagate { + assert_eq!(peers.len(), 2); + assert_eq!(assignments.len(), 1); + } + ); + + // send the an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices, + validator: validator_index, + signature: dummy_signature(), + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_b, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, + tx, + )) => { + assert_eq!(vote, approval); + tx.send(ApprovalCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert_eq!(peers.len(), 1); + assert_eq!(approvals.len(), 1); + } + ); + virtual_overseer + }); +} + +// Tests that votes that cover multiple assignments candidates are correctly processed on importing +#[test] +fn multiple_assignments_covered_with_one_approval_vote() { + let peers = make_peers_and_authority_ids(15); + + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let peer_d = peers.get(4).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + // setup peers with V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::VStaging).await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 2], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // Set up a gossip topology, where a, b, and c, d are topology neighboors to the node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(2); // peer_c is the originator + let candidate_indices: CandidateBitfield = + vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); + + let core_bitfields = vec![CoreIndex(0)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfields); + + // send the candidate 0 assignment from peer_b + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (0 as CandidateIndex).into(), + )]); + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tx, + )) => { + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() >= 2); + assert!(peers.contains(&peer_a)); + assert!(peers.contains(&peer_b)); + assert_eq!(assignments.len(), 1); + } + ); + + let candidate_bitfields = vec![CoreIndex(1)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); + + // send the candidate 1 assignment from peer_c + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (1 as CandidateIndex).into(), + )]); + + send_message_from_peer_v2(overseer, &peer_c, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tx, + )) => { + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + expect_reputation_change(overseer, &peer_c, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() >= 2); + assert!(peers.contains(&peer_b)); + assert!(peers.contains(&peer_a)); + assert_eq!(assignments.len(), 1); + } + ); + + // send an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices, + validator: validator_index, + signature: dummy_signature(), + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, + tx, + )) => { + assert_eq!(vote, approval); + tx.send(ApprovalCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert!(peers.len() >= 2); + assert!(peers.contains(&peer_b)); + assert!(peers.contains(&peer_a)); + assert_eq!(approvals.len(), 1); + } + ); + for candidate_index in 0..1 { + let (tx_distribution, rx_distribution) = oneshot::channel(); + let mut candidates_requesting_signatures = HashSet::new(); + candidates_requesting_signatures.insert((hash, candidate_index)); + overseer_send( + overseer, + ApprovalDistributionMessage::GetApprovalSignatures( + candidates_requesting_signatures, + tx_distribution, + ), + ) + .await; + let signatures = rx_distribution.await.unwrap(); + + assert_eq!(signatures.len(), 1); + for (signing_validator, signature) in signatures { + assert_eq!(validator_index, signing_validator); + assert_eq!(signature.0, hash); + assert_eq!(signature.2, approval.signature); + assert_eq!(signature.1, vec![0, 1]); + } + } + virtual_overseer + }); +} + +// Tests that votes that cover multiple assignments candidates are correctly processed when unify +// with peer view +#[test] +fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { + let peers = make_peers_and_authority_ids(15); + + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_d = peers.get(4).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); + let hash = Hash::repeat_byte(0xAA); + + let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { + let overseer = &mut virtual_overseer; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::VStaging).await; + + // new block `hash_a` with 1 candidates + let meta = BlockApprovalMeta { + hash, + parent_hash, + number: 1, + candidates: vec![Default::default(); 2], + slot: 1.into(), + session: 1, + }; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); + overseer_send(overseer, msg).await; + + // Set up a gossip topology, where a, b, and c, d are topology neighboors to the node. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + + // import an assignment related to `hash` locally + let validator_index = ValidatorIndex(2); // peer_c is the originator + let candidate_indices: CandidateBitfield = + vec![0 as CandidateIndex, 1 as CandidateIndex].try_into().unwrap(); + + let core_bitfields = vec![CoreIndex(0)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfields); + + // send the candidate 0 assignment from peer_b + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (0 as CandidateIndex).into(), + )]); + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tx, + )) => { + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + let candidate_bitfields = vec![CoreIndex(1)].try_into().unwrap(); + let cert = fake_assignment_cert_v2(hash, validator_index, candidate_bitfields); + + // send the candidate 1 assignment from peer_c + let assignment = IndirectAssignmentCertV2 { + block_hash: hash, + validator: validator_index, + cert: cert.cert, + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + assignment, + (1 as CandidateIndex).into(), + )]); + + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + _, _, + tx, + )) => { + tx.send(AssignmentCheckResult::Accepted).unwrap(); + } + ); + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + // send an approval from peer_b + let approval = IndirectSignedApprovalVoteV2 { + block_hash: hash, + candidate_indices, + validator: validator_index, + signature: dummy_signature(), + }; + let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v2(overseer, &peer_d, msg).await; + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + vote, + tx, + )) => { + assert_eq!(vote, approval); + tx.send(ApprovalCheckResult::Accepted).unwrap(); + } + ); + + expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; + + // setup peers with V2 protocol versions + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + let mut expected_peers_assignments = vec![peer_a, peer_b]; + let mut expected_peers_approvals = vec![peer_a, peer_b]; + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_assignments.contains(peers.first().unwrap())); + expected_peers_assignments.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(assignments.len(), 2); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_approvals.contains(peers.first().unwrap())); + expected_peers_approvals.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(approvals.len(), 1); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_assignments.contains(peers.first().unwrap())); + expected_peers_assignments.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(assignments.len(), 2); + } + ); + + assert_matches!( + overseer_recv(overseer).await, + AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( + peers, + Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( + protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + )) + )) => { + assert!(peers.len() == 1); + assert!(expected_peers_approvals.contains(peers.first().unwrap())); + expected_peers_approvals.retain(|peer| peer != peers.first().unwrap()); + assert_eq!(approvals.len(), 1); + } + ); + + virtual_overseer + }); +} + #[test] fn import_approval_bad() { let peer_a = PeerId::random(); From da61d985a863c8ccfefaf000d2e871a3433e71b0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 1 Sep 2023 09:17:09 +0300 Subject: [PATCH 14/89] Revert "Log errors when banning peers" This reverts commit 5e004e16e23525fbfe423b4ea15cd36d750e5646. --- substrate/client/network/src/peer_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/network/src/peer_store.rs b/substrate/client/network/src/peer_store.rs index 96bcd47cc12e..2f3d4a1fd1a0 100644 --- a/substrate/client/network/src/peer_store.rs +++ b/substrate/client/network/src/peer_store.rs @@ -222,7 +222,7 @@ impl PeerStoreInner { if peer_info.reputation < BANNED_THRESHOLD { self.protocols.iter().for_each(|handle| handle.disconnect_peer(peer_id)); - log::error!( + log::trace!( target: LOG_TARGET, "Report {}: {:+} to {}. Reason: {}. Banned, disconnecting.", peer_id, From f3fee248ace628198101f1d29946af644a215651 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 1 Sep 2023 10:10:44 +0300 Subject: [PATCH 15/89] Cleanup post migrating hacks when migrating from polkadot repo Signed-off-by: Alexandru Gheorghe --- .../src/blockchain_rpc_client.rs | 5 +- .../src/rpc_client.rs | 15 +- .../network/availability-recovery/src/lib.rs | 2 +- .../src/legacy_v1/mod.rs | 4 - polkadot/node/service/src/fake_runtime_api.rs | 1 - polkadot/primitives/src/runtime_api.rs | 1 + polkadot/primitives/src/v5/mod.rs | 15 +- polkadot/runtime/kusama/src/lib.rs | 1 - .../src/configuration/migration/v7.rs | 1 - polkadot/runtime/polkadot/src/lib.rs | 1 - polkadot/runtime/rococo/src/lib.rs | 1 - polkadot/scripts/ci/gitlab/pipeline/build.yml | 174 ------ .../scripts/ci/gitlab/pipeline/zombienet.yml | 502 ------------------ 13 files changed, 23 insertions(+), 700 deletions(-) delete mode 100644 polkadot/scripts/ci/gitlab/pipeline/build.yml delete mode 100644 polkadot/scripts/ci/gitlab/pipeline/zombienet.yml diff --git a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs index 855d0bf65c77..a40edc0ed3b1 100644 --- a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs +++ b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs @@ -357,9 +357,10 @@ impl RuntimeApiSubsystemClient for BlockChainRpcClient { ) -> Result, ApiError> { Ok(self.rpc_client.parachain_host_staging_para_backing_state(at, para_id).await?) } + /// Approval voting configuration parameters - async fn approval_voting_params(&self, _at: Hash) -> Result { - Ok(ApprovalVotingParams { max_approval_coalesce_count: 1 }) + async fn approval_voting_params(&self, at: Hash) -> Result { + Ok(self.rpc_client.parachain_host_staging_approval_voting_params(at).await?) } } diff --git a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs index c1e92b249d77..a8792b7f8127 100644 --- a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs +++ b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs @@ -31,7 +31,7 @@ use parity_scale_codec::{Decode, Encode}; use cumulus_primitives_core::{ relay_chain::{ slashing, - vstaging::{AsyncBackingParams, BackingState}, + vstaging::{ApprovalVotingParams, AsyncBackingParams, BackingState}, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash as RelayHash, Header as RelayHeader, InboundHrmpMessage, OccupiedCoreAssumption, @@ -611,6 +611,19 @@ impl RelayChainRpcClient { .await } + #[allow(missing_docs)] + pub async fn parachain_host_staging_approval_voting_params( + &self, + at: RelayHash, + ) -> Result { + self.call_remote_runtime_function( + "ParachainHost_staging_approval_voting_params", + at, + None::<()>, + ) + .await + } + #[allow(missing_docs)] pub async fn parachain_host_staging_para_backing_state( &self, diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index f765e5680568..e40f8ee1ea44 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -102,7 +102,7 @@ const TIMEOUT_START_NEW_REQUESTS: Duration = Duration::from_millis(100); /// PoV size limit in bytes for which prefer fetching from backers. //TODO: Cleanup increased for versi testing -const SMALL_POV_LIMIT: usize = 3 * 1024 * 1024; +const SMALL_POV_LIMIT: usize = 128 * 1024; #[derive(Clone, PartialEq)] /// The strategy we use to recover the PoV. diff --git a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs index 082b777e0270..9ae76047383c 100644 --- a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs +++ b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs @@ -1195,10 +1195,6 @@ async fn modify_reputation( peer: PeerId, rep: Rep, ) { - //TODO: Test teory in versi - if rep == COST_DUPLICATE_STATEMENT { - return - } reputation.modify(sender, peer, rep).await; } diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs index a8ec22baf791..d9553afa024b 100644 --- a/polkadot/node/service/src/fake_runtime_api.rs +++ b/polkadot/node/service/src/fake_runtime_api.rs @@ -116,7 +116,6 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] impl runtime_api::ParachainHost for Runtime { fn validators() -> Vec { unimplemented!() diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index 30e0ec0b4bda..f275682b50bd 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -241,6 +241,7 @@ sp_api::decl_runtime_apis! { key_ownership_proof: vstaging::slashing::OpaqueKeyOwnershipProof, ) -> Option<()>; + /***** Staging *****/ /// Get the minimum number of backing votes for a parachain candidate. /// This is a staging method! Do not use on production runtimes! diff --git a/polkadot/primitives/src/v5/mod.rs b/polkadot/primitives/src/v5/mod.rs index c190d406efa3..55e4b0481775 100644 --- a/polkadot/primitives/src/v5/mod.rs +++ b/polkadot/primitives/src/v5/mod.rs @@ -354,13 +354,6 @@ pub mod well_known_keys { /// Unique identifier for the Parachains Inherent pub const PARACHAINS_INHERENT_IDENTIFIER: InherentIdentifier = *b"parachn0"; -// /// TODO: Make this two a parachain host configuration. -// /// Maximum allowed candidates to be signed withing a signle approval votes. -// pub const MAX_APPROVAL_COALESCE_COUNT: u64 = 6; -// /// The maximum time we await for an approval to be coalesced with other approvals -// /// before we sign it and distribute to our peers -// pub const MAX_APPROVAL_COALESCE_WAIT_MILLIS: u64 = 500; - /// The key type ID for parachain assignment key. pub const ASSIGNMENT_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"asgn"); @@ -1134,10 +1127,10 @@ impl<'a> ApprovalVoteMultipleCandidates<'a> { /// Yields the signing payload for this approval vote. pub fn signing_payload(&self, session_index: SessionIndex) -> Vec { const MAGIC: [u8; 4] = *b"APPR"; - // Make this backwards compatible with `ApprovalVote` so if we have just on candidate the signature - // will look the same. - // This gives us the nice benefit that old nodes can still check signatures when len is 1 and the - // new node can check the signature coming from old nodes. + // Make this backwards compatible with `ApprovalVote` so if we have just on candidate the + // signature will look the same. + // This gives us the nice benefit that old nodes can still check signatures when len is 1 + // and the new node can check the signature coming from old nodes. if self.0.len() == 1 { (MAGIC, self.0.first().expect("QED: we just checked"), session_index).encode() } else { diff --git a/polkadot/runtime/kusama/src/lib.rs b/polkadot/runtime/kusama/src/lib.rs index e20f898de75c..4b5f03b38c6c 100644 --- a/polkadot/runtime/kusama/src/lib.rs +++ b/polkadot/runtime/kusama/src/lib.rs @@ -1885,7 +1885,6 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/parachains/src/configuration/migration/v7.rs b/polkadot/runtime/parachains/src/configuration/migration/v7.rs index 3624eb982323..113651381207 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v7.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v7.rs @@ -240,7 +240,6 @@ pvf_voting_ttl : pre.pvf_voting_ttl, minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, async_backing_params : pre.async_backing_params, executor_params : pre.executor_params, - } }; diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index bd1890a58f75..2efd329eea0c 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -1688,7 +1688,6 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index b3897f06a062..a80f45c340d9 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1711,7 +1711,6 @@ sp_api::impl_runtime_apis! { } } - #[api_version(5)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() diff --git a/polkadot/scripts/ci/gitlab/pipeline/build.yml b/polkadot/scripts/ci/gitlab/pipeline/build.yml deleted file mode 100644 index 70d1b1fc5456..000000000000 --- a/polkadot/scripts/ci/gitlab/pipeline/build.yml +++ /dev/null @@ -1,174 +0,0 @@ -# This file is part of .gitlab-ci.yml -# Here are all jobs that are executed during "build" stage - -build-linux-stable: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .common-refs - - .compiler-info - - .collect-artifacts - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - # Ensure we run the UI tests. - RUN_UI_TESTS: 1 - script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime,network-protocol-staging --verbose --bins - # pack artifacts - - mkdir -p ./artifacts - - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name - - mv ./target/testnet/polkadot ./artifacts/. - - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. - - mv ./target/testnet/polkadot-execute-worker ./artifacts/. - - pushd artifacts - - sha256sum polkadot | tee polkadot.sha256 - - shasum -c polkadot.sha256 - - popd - - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" - - echo -n ${VERSION} > ./artifacts/VERSION - - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG - - echo -n ${CI_JOB_ID} > ./artifacts/BUILD_LINUX_JOB_ID - - RELEASE_VERSION=$(./artifacts/polkadot -V | awk '{print $2}'| awk -F "-" '{print $1}') - - echo -n "v${RELEASE_VERSION}" > ./artifacts/BUILD_RELEASE_VERSION - -build-test-collators: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .common-refs - - .compiler-info - - .collect-artifacts - script: - - time cargo build --locked --profile testnet --verbose -p test-parachain-adder-collator - - time cargo build --locked --profile testnet --verbose -p test-parachain-undying-collator - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/adder-collator ./artifacts/. - - mv ./target/testnet/undying-collator ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - -build-malus: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .common-refs - - .compiler-info - - .collect-artifacts - script: - - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/malus ./artifacts/. - - mv ./target/testnet/polkadot-execute-worker ./artifacts/. - - mv ./target/testnet/polkadot-prepare-worker ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - -build-staking-miner: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .common-refs - - .compiler-info - - .collect-artifacts - script: - - time cargo build --locked --release --package staking-miner - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/release/staking-miner ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - -build-rustdoc: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in test.yml - needs: - - job: test-deterministic-wasm - artifacts: false - extends: - - .docker-env - - .test-refs - variables: - SKIP_WASM_BUILD: 1 - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - when: on_success - expire_in: 1 days - paths: - - ./crate-docs/ - script: - # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` - # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable - - time cargo doc --workspace --verbose --no-deps - - rm -f ./target/doc/.lock - - mv ./target/doc ./crate-docs - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot ./crate-docs - - echo "" > ./crate-docs/index.html - -build-implementers-guide: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in test.yml - needs: - - job: test-deterministic-wasm - artifacts: false - extends: - - .kubernetes-env - - .test-refs - - .collect-artifacts-short - # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 - variables: - GIT_STRATEGY: clone - GIT_DEPTH: 0 - CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123 - script: - - mdbook build ./roadmap/implementers-guide - - mkdir -p artifacts - - mv roadmap/implementers-guide/book artifacts/ - -build-short-benchmark: - stage: build - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # the job can be found in check.yml - needs: - - job: job-starter - artifacts: false - extends: - - .docker-env - - .test-refs - - .collect-artifacts - script: - - cargo build --profile release --locked --features=runtime-benchmarks - - mkdir artifacts - - cp ./target/release/polkadot ./artifacts/ diff --git a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml deleted file mode 100644 index 2384a5935661..000000000000 --- a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml +++ /dev/null @@ -1,502 +0,0 @@ -# This file is part of .gitlab-ci.yml -# Here are all jobs that are executed during "zombienet" stage - -zombienet-tests-parachains-smoke-test: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE="docker.io/paritypr/colander:7292" # The collator image is fixed - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-smoke-test.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-pvf: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-pvf.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-approval-coalescing: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0006-approval-voting-coalescing.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-disputes.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes-garbage-candidate: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-parachains-garbage-candidate.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes-past-session: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0004-parachains-disputes-past-session.zndsl" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-max-tranche0-approvals: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0005-parachains-max-tranche0.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-test-parachains-upgrade-smoke-test: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "ZombieNet Tests Config" - - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" - - echo "docker.io/parity/polkadot-collator:latest" - - echo "${ZOMBIENET_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE="docker.io/parity/polkadot-parachain:latest" # Use cumulus lastest image - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-upgrade-smoke-test.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-misc-paritydb: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-paritydb.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-misc-upgrade-node: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: build-linux-stable - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/parity/polkadot:latest" - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - - BUILD_LINUX_JOB_ID="$(cat ./artifacts/BUILD_LINUX_JOB_ID)" - - export POLKADOT_PR_ARTIFACTS_URL="https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/${BUILD_LINUX_JOB_ID}/artifacts/raw/artifacts" - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-upgrade-node.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-malus-dispute-valid: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-dispute-valid-block.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-deregister-register-validator: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-deregister-register-validator-smoke.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-beefy-and-mmr: - stage: zombienet - image: "${ZOMBIENET_IMAGE}" - extends: - - .kubernetes-env - - .zombienet-refs - needs: - - job: publish-polkadot-debug-image - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-beefy-and-mmr.zndsl" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-async-backing-compatibility: - stage: zombienet - extends: - - .kubernetes-env - - .zombienet-refs - image: "${ZOMBIENET_IMAGE}" - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: build-linux-stable - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/async_backing" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="001-async-backing-compatibility.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-async-backing-runtime-upgrade: - stage: zombienet - extends: - - .kubernetes-env - - .zombienet-refs - image: "${ZOMBIENET_IMAGE}" - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: build-linux-stable - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/async_backing" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - - export POLKADOT_PR_BIN_URL="https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/${BUILD_LINUX_JOB_ID}/artifacts/raw/artifacts/polkadot" - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="002-async-backing-runtime-upgrade.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-async-backing-collator-mix: - stage: zombienet - extends: - - .kubernetes-env - - .zombienet-refs - image: "${ZOMBIENET_IMAGE}" - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: build-linux-stable - artifacts: true - variables: - RUN_IN_CONTAINER: "1" - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/async_backing" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="003-async-backing-collator-mix.zndsl" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test From 6338d3301c7b9bda2ef03ea4405e51a3e666863e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 1 Sep 2023 12:04:33 +0300 Subject: [PATCH 16/89] Fixup clippy Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/approval-distribution/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 8d28a54dcd89..9b0d42a5ff64 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -596,8 +596,7 @@ impl BlockEntry { .filter_map(|(validator, assignment_bitfield)| { self.approval_entries.get(&(*validator, assignment_bitfield.clone())) }) - .map(|approval_entry| approval_entry.approvals.clone().into_iter()) - .flatten() + .flat_map(|approval_entry| approval_entry.approvals.clone().into_iter()) .collect() }); From 7fdae09a5c1e46fa4d9b520d356c18bb3049725e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 26 Sep 2023 15:34:57 +0300 Subject: [PATCH 17/89] Add host configuration v10 with approval_voting params ... the param was incorrectly appended to v9 instead of creating a new version as v10. Signed-off-by: Alexandru Gheorghe --- .../runtime/parachains/src/configuration.rs | 1 - .../parachains/src/configuration/migration.rs | 1 + .../src/configuration/migration/v10.rs | 327 ++++++++++++++++++ .../src/configuration/migration/v9.rs | 110 +++++- polkadot/runtime/polkadot/src/lib.rs | 1 + polkadot/runtime/rococo/src/lib.rs | 1 + polkadot/runtime/westend/src/lib.rs | 1 + 7 files changed, 436 insertions(+), 6 deletions(-) create mode 100644 polkadot/runtime/parachains/src/configuration/migration/v10.rs diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index b0e5810ac686..6690e4156c7f 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -261,7 +261,6 @@ pub struct HostConfiguration { /// backable. pub minimum_backing_votes: u32, /// Params used by approval-voting - /// TODO: fixme this is not correctly migrated pub approval_voting_params: ApprovalVotingParams, } diff --git a/polkadot/runtime/parachains/src/configuration/migration.rs b/polkadot/runtime/parachains/src/configuration/migration.rs index 26f8a85b496d..db323d3aad93 100644 --- a/polkadot/runtime/parachains/src/configuration/migration.rs +++ b/polkadot/runtime/parachains/src/configuration/migration.rs @@ -16,6 +16,7 @@ //! A module that is responsible for migration of storage. +pub mod v10; pub mod v6; pub mod v7; pub mod v8; diff --git a/polkadot/runtime/parachains/src/configuration/migration/v10.rs b/polkadot/runtime/parachains/src/configuration/migration/v10.rs new file mode 100644 index 000000000000..53c5f4d2c52c --- /dev/null +++ b/polkadot/runtime/parachains/src/configuration/migration/v10.rs @@ -0,0 +1,327 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! A module that is responsible for migration of storage. + +use crate::configuration::{self, Config, Pallet}; +use frame_support::{ + pallet_prelude::*, + traits::{Defensive, StorageVersion}, + weights::Weight, +}; +use frame_system::pallet_prelude::BlockNumberFor; +use primitives::{vstaging::ApprovalVotingParams, SessionIndex, LEGACY_MIN_BACKING_VOTES}; +use sp_runtime::Perbill; +use sp_std::vec::Vec; + +use frame_support::traits::OnRuntimeUpgrade; + +use super::v9::V9HostConfiguration; +type V10HostConfiguration = configuration::HostConfiguration; + +mod v9 { + use super::*; + + #[frame_support::storage_alias] + pub(crate) type ActiveConfig = + StorageValue, V9HostConfiguration>, OptionQuery>; + + #[frame_support::storage_alias] + pub(crate) type PendingConfigs = StorageValue< + Pallet, + Vec<(SessionIndex, V9HostConfiguration>)>, + OptionQuery, + >; +} + +mod v10 { + use super::*; + + #[frame_support::storage_alias] + pub(crate) type ActiveConfig = + StorageValue, V10HostConfiguration>, OptionQuery>; + + #[frame_support::storage_alias] + pub(crate) type PendingConfigs = StorageValue< + Pallet, + Vec<(SessionIndex, V10HostConfiguration>)>, + OptionQuery, + >; +} + +pub struct MigrateToV10(sp_std::marker::PhantomData); +impl OnRuntimeUpgrade for MigrateToV10 { + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { + log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV10"); + Ok(Vec::new()) + } + + fn on_runtime_upgrade() -> Weight { + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 started"); + if StorageVersion::get::>() == 9 { + let weight_consumed = migrate_to_v10::(); + + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 executed successfully"); + StorageVersion::new(10).put::>(); + + weight_consumed + } else { + log::warn!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 should be removed."); + T::DbWeight::get().reads(1) + } + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + log::trace!(target: crate::configuration::LOG_TARGET, "Running post_upgrade() for HostConfiguration MigrateToV10"); + ensure!( + StorageVersion::get::>() >= 10, + "Storage version should be >= 10 after the migration" + ); + + Ok(()) + } +} + +fn migrate_to_v10() -> Weight { + // Unusual formatting is justified: + // - make it easier to verify that fields assign what they supposed to assign. + // - this code is transient and will be removed after all migrations are done. + // - this code is important enough to optimize for legibility sacrificing consistency. + #[rustfmt::skip] + let translate = + |pre: V9HostConfiguration>| -> + V10HostConfiguration> + { + V10HostConfiguration { +max_code_size : pre.max_code_size, +max_head_data_size : pre.max_head_data_size, +max_upward_queue_count : pre.max_upward_queue_count, +max_upward_queue_size : pre.max_upward_queue_size, +max_upward_message_size : pre.max_upward_message_size, +max_upward_message_num_per_candidate : pre.max_upward_message_num_per_candidate, +hrmp_max_message_num_per_candidate : pre.hrmp_max_message_num_per_candidate, +validation_upgrade_cooldown : pre.validation_upgrade_cooldown, +validation_upgrade_delay : pre.validation_upgrade_delay, +max_pov_size : pre.max_pov_size, +max_downward_message_size : pre.max_downward_message_size, +hrmp_sender_deposit : pre.hrmp_sender_deposit, +hrmp_recipient_deposit : pre.hrmp_recipient_deposit, +hrmp_channel_max_capacity : pre.hrmp_channel_max_capacity, +hrmp_channel_max_total_size : pre.hrmp_channel_max_total_size, +hrmp_max_parachain_inbound_channels : pre.hrmp_max_parachain_inbound_channels, +hrmp_max_parachain_outbound_channels : pre.hrmp_max_parachain_outbound_channels, +hrmp_channel_max_message_size : pre.hrmp_channel_max_message_size, +code_retention_period : pre.code_retention_period, +on_demand_cores : pre.on_demand_cores, +on_demand_retries : pre.on_demand_retries, +group_rotation_frequency : pre.group_rotation_frequency, +paras_availability_period : pre.paras_availability_period, +scheduling_lookahead : pre.scheduling_lookahead, +max_validators_per_core : pre.max_validators_per_core, +max_validators : pre.max_validators, +dispute_period : pre.dispute_period, +dispute_post_conclusion_acceptance_period: pre.dispute_post_conclusion_acceptance_period, +no_show_slots : pre.no_show_slots, +n_delay_tranches : pre.n_delay_tranches, +zeroth_delay_tranche_width : pre.zeroth_delay_tranche_width, +needed_approvals : pre.needed_approvals, +relay_vrf_modulo_samples : pre.relay_vrf_modulo_samples, +pvf_voting_ttl : pre.pvf_voting_ttl, +minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, +async_backing_params : pre.async_backing_params, +executor_params : pre.executor_params, +on_demand_queue_max_size : 10_000u32, +on_demand_base_fee : 10_000_000u128, +on_demand_fee_variability : Perbill::from_percent(3), +on_demand_target_queue_utilization : Perbill::from_percent(25), +on_demand_ttl : 5u32.into(), +minimum_backing_votes : LEGACY_MIN_BACKING_VOTES, +approval_voting_params : ApprovalVotingParams { + max_approval_coalesce_count: 1, + } + } + }; + + let v9 = v9::ActiveConfig::::get() + .defensive_proof("Could not decode old config") + .unwrap_or_default(); + let v10 = translate(v9); + v10::ActiveConfig::::set(Some(v10)); + + // Allowed to be empty. + let pending_v9 = v9::PendingConfigs::::get().unwrap_or_default(); + let mut pending_v10 = Vec::new(); + + for (session, v9) in pending_v9.into_iter() { + let v10 = translate(v9); + pending_v10.push((session, v10)); + } + v10::PendingConfigs::::set(Some(pending_v10.clone())); + + let num_configs = (pending_v10.len() + 1) as u64; + T::DbWeight::get().reads_writes(num_configs, num_configs) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::{new_test_ext, Test}; + + #[test] + fn v10_deserialized_from_actual_data() { + // Example how to get new `raw_config`: + // We'll obtain the raw_config at a specified a block + // Steps: + // 1. Go to Polkadot.js -> Developer -> Chain state -> Storage: https://polkadot.js.org/apps/#/chainstate + // 2. Set these parameters: + // 2.1. selected state query: configuration; activeConfig(): + // PolkadotRuntimeParachainsConfigurationHostConfiguration + // 2.2. blockhash to query at: + // 0xf89d3ab5312c5f70d396dc59612f0aa65806c798346f9db4b35278baed2e0e53 (the hash of + // the block) + // 2.3. Note the value of encoded storage key -> + // 0x06de3d8a54d27e44a9d5ce189618f22db4b49d95320d9021994c850f25b8e385 for the + // referenced block. + // 2.4. You'll also need the decoded values to update the test. + // 3. Go to Polkadot.js -> Developer -> Chain state -> Raw storage + // 3.1 Enter the encoded storage key and you get the raw config. + + // This exceeds the maximal line width length, but that's fine, since this is not code and + // doesn't need to be read and also leaving it as one line allows to easily copy it. + let raw_config = + hex_literal::hex![" + 0000300000800000080000000000100000c8000005000000050000000200000002000000000000000000000000005000000010000400000000000000000000000000000000000000000000000000000000000000000000000800000000200000040000000000100000b004000000000000000000001027000080b2e60e80c3c90180969800000000000000000000000000050000001400000004000000010000000101000000000600000064000000020000001900000000000000030000000200000002000000050000000200000001000000" + ]; + + let v10 = + V10HostConfiguration::::decode(&mut &raw_config[..]).unwrap(); + + // We check only a sample of the values here. If we missed any fields or messed up data + // types that would skew all the fields coming after. + assert_eq!(v10.max_code_size, 3_145_728); + assert_eq!(v10.validation_upgrade_cooldown, 2); + assert_eq!(v10.max_pov_size, 5_242_880); + assert_eq!(v10.hrmp_channel_max_message_size, 1_048_576); + assert_eq!(v10.n_delay_tranches, 25); + assert_eq!(v10.minimum_validation_upgrade_delay, 5); + assert_eq!(v10.group_rotation_frequency, 20); + assert_eq!(v10.on_demand_cores, 0); + assert_eq!(v10.on_demand_base_fee, 10_000_000); + assert_eq!(v10.minimum_backing_votes, LEGACY_MIN_BACKING_VOTES); + assert_eq!(v10.approval_voting_params.max_approval_coalesce_count, 1); + } + + #[test] + fn test_migrate_to_v10() { + // Host configuration has lots of fields. However, in this migration we only add one + // field. The most important part to check are a couple of the last fields. We also pick + // extra fields to check arbitrarily, e.g. depending on their position (i.e. the middle) and + // also their type. + // + // We specify only the picked fields and the rest should be provided by the `Default` + // implementation. That implementation is copied over between the two types and should work + // fine. + let v9 = V9HostConfiguration:: { + needed_approvals: 69, + paras_availability_period: 55, + hrmp_recipient_deposit: 1337, + max_pov_size: 1111, + minimum_validation_upgrade_delay: 20, + ..Default::default() + }; + + let mut pending_configs = Vec::new(); + pending_configs.push((100, v9.clone())); + pending_configs.push((300, v9.clone())); + + new_test_ext(Default::default()).execute_with(|| { + // Implant the v9 version in the state. + v9::ActiveConfig::::set(Some(v9)); + v9::PendingConfigs::::set(Some(pending_configs)); + + migrate_to_v10::(); + + let v10 = v10::ActiveConfig::::get().unwrap(); + assert_eq!(v10.approval_voting_params.max_approval_coalesce_count, 1); + + let mut configs_to_check = v10::PendingConfigs::::get().unwrap(); + configs_to_check.push((0, v10.clone())); + + for (_, v9) in configs_to_check { + #[rustfmt::skip] + { + assert_eq!(v9.max_code_size , v10.max_code_size); + assert_eq!(v9.max_head_data_size , v10.max_head_data_size); + assert_eq!(v9.max_upward_queue_count , v10.max_upward_queue_count); + assert_eq!(v9.max_upward_queue_size , v10.max_upward_queue_size); + assert_eq!(v9.max_upward_message_size , v10.max_upward_message_size); + assert_eq!(v9.max_upward_message_num_per_candidate , v10.max_upward_message_num_per_candidate); + assert_eq!(v9.hrmp_max_message_num_per_candidate , v10.hrmp_max_message_num_per_candidate); + assert_eq!(v9.validation_upgrade_cooldown , v10.validation_upgrade_cooldown); + assert_eq!(v9.validation_upgrade_delay , v10.validation_upgrade_delay); + assert_eq!(v9.max_pov_size , v10.max_pov_size); + assert_eq!(v9.max_downward_message_size , v10.max_downward_message_size); + assert_eq!(v9.hrmp_max_parachain_outbound_channels , v10.hrmp_max_parachain_outbound_channels); + assert_eq!(v9.hrmp_sender_deposit , v10.hrmp_sender_deposit); + assert_eq!(v9.hrmp_recipient_deposit , v10.hrmp_recipient_deposit); + assert_eq!(v9.hrmp_channel_max_capacity , v10.hrmp_channel_max_capacity); + assert_eq!(v9.hrmp_channel_max_total_size , v10.hrmp_channel_max_total_size); + assert_eq!(v9.hrmp_max_parachain_inbound_channels , v10.hrmp_max_parachain_inbound_channels); + assert_eq!(v9.hrmp_channel_max_message_size , v10.hrmp_channel_max_message_size); + assert_eq!(v9.code_retention_period , v10.code_retention_period); + assert_eq!(v9.on_demand_cores , v10.on_demand_cores); + assert_eq!(v9.on_demand_retries , v10.on_demand_retries); + assert_eq!(v9.group_rotation_frequency , v10.group_rotation_frequency); + assert_eq!(v9.paras_availability_period , v10.paras_availability_period); + assert_eq!(v9.scheduling_lookahead , v10.scheduling_lookahead); + assert_eq!(v9.max_validators_per_core , v10.max_validators_per_core); + assert_eq!(v9.max_validators , v10.max_validators); + assert_eq!(v9.dispute_period , v10.dispute_period); + assert_eq!(v9.no_show_slots , v10.no_show_slots); + assert_eq!(v9.n_delay_tranches , v10.n_delay_tranches); + assert_eq!(v9.zeroth_delay_tranche_width , v10.zeroth_delay_tranche_width); + assert_eq!(v9.needed_approvals , v10.needed_approvals); + assert_eq!(v9.relay_vrf_modulo_samples , v10.relay_vrf_modulo_samples); + assert_eq!(v9.pvf_voting_ttl , v10.pvf_voting_ttl); + assert_eq!(v9.minimum_validation_upgrade_delay , v10.minimum_validation_upgrade_delay); + assert_eq!(v9.async_backing_params.allowed_ancestry_len, v10.async_backing_params.allowed_ancestry_len); + assert_eq!(v9.async_backing_params.max_candidate_depth , v10.async_backing_params.max_candidate_depth); + assert_eq!(v9.executor_params , v10.executor_params); + assert_eq!(v9.minimum_backing_votes , v10.minimum_backing_votes); + }; // ; makes this a statement. `rustfmt::skip` cannot be put on an expression. + } + }); + } + + // Test that migration doesn't panic in case there're no pending configurations upgrades in + // pallet's storage. + #[test] + fn test_migrate_to_v10_no_pending() { + let v9 = V9HostConfiguration::::default(); + + new_test_ext(Default::default()).execute_with(|| { + // Implant the v9 version in the state. + v9::ActiveConfig::::set(Some(v9)); + // Ensure there're no pending configs. + v10::PendingConfigs::::set(None); + + // Shouldn't fail. + migrate_to_v10::(); + }); + } +} diff --git a/polkadot/runtime/parachains/src/configuration/migration/v9.rs b/polkadot/runtime/parachains/src/configuration/migration/v9.rs index 37cb451ab36b..909a63ae6291 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v9.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v9.rs @@ -23,14 +23,117 @@ use frame_support::{ weights::Weight, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::{vstaging::ApprovalVotingParams, SessionIndex, LEGACY_MIN_BACKING_VOTES}; +use primitives::{ + vstaging::AsyncBackingParams, Balance, ExecutorParams, SessionIndex, LEGACY_MIN_BACKING_VOTES, + ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, +}; + use sp_runtime::Perbill; use sp_std::vec::Vec; use frame_support::traits::OnRuntimeUpgrade; use super::v8::V8HostConfiguration; -type V9HostConfiguration = configuration::HostConfiguration; +/// All configuration of the runtime with respect to paras. +#[derive(Clone, Encode, Decode, Debug)] +pub struct V9HostConfiguration { + pub max_code_size: u32, + pub max_head_data_size: u32, + pub max_upward_queue_count: u32, + pub max_upward_queue_size: u32, + pub max_upward_message_size: u32, + pub max_upward_message_num_per_candidate: u32, + pub hrmp_max_message_num_per_candidate: u32, + pub validation_upgrade_cooldown: BlockNumber, + pub validation_upgrade_delay: BlockNumber, + pub async_backing_params: AsyncBackingParams, + pub max_pov_size: u32, + pub max_downward_message_size: u32, + pub hrmp_max_parachain_outbound_channels: u32, + pub hrmp_sender_deposit: Balance, + pub hrmp_recipient_deposit: Balance, + pub hrmp_channel_max_capacity: u32, + pub hrmp_channel_max_total_size: u32, + pub hrmp_max_parachain_inbound_channels: u32, + pub hrmp_channel_max_message_size: u32, + pub executor_params: ExecutorParams, + pub code_retention_period: BlockNumber, + pub on_demand_cores: u32, + pub on_demand_retries: u32, + pub on_demand_queue_max_size: u32, + pub on_demand_target_queue_utilization: Perbill, + pub on_demand_fee_variability: Perbill, + pub on_demand_base_fee: Balance, + pub on_demand_ttl: BlockNumber, + pub group_rotation_frequency: BlockNumber, + pub paras_availability_period: BlockNumber, + pub scheduling_lookahead: u32, + pub max_validators_per_core: Option, + pub max_validators: Option, + pub dispute_period: SessionIndex, + pub dispute_post_conclusion_acceptance_period: BlockNumber, + pub no_show_slots: u32, + pub n_delay_tranches: u32, + pub zeroth_delay_tranche_width: u32, + pub needed_approvals: u32, + pub relay_vrf_modulo_samples: u32, + pub pvf_voting_ttl: SessionIndex, + pub minimum_validation_upgrade_delay: BlockNumber, + pub minimum_backing_votes: u32, +} + +impl> Default for V9HostConfiguration { + fn default() -> Self { + Self { + async_backing_params: AsyncBackingParams { + max_candidate_depth: 0, + allowed_ancestry_len: 0, + }, + group_rotation_frequency: 1u32.into(), + paras_availability_period: 1u32.into(), + no_show_slots: 1u32.into(), + validation_upgrade_cooldown: Default::default(), + validation_upgrade_delay: 2u32.into(), + code_retention_period: Default::default(), + max_code_size: Default::default(), + max_pov_size: Default::default(), + max_head_data_size: Default::default(), + on_demand_cores: Default::default(), + on_demand_retries: Default::default(), + scheduling_lookahead: 1, + max_validators_per_core: Default::default(), + max_validators: None, + dispute_period: 6, + dispute_post_conclusion_acceptance_period: 100.into(), + n_delay_tranches: Default::default(), + zeroth_delay_tranche_width: Default::default(), + needed_approvals: Default::default(), + relay_vrf_modulo_samples: Default::default(), + max_upward_queue_count: Default::default(), + max_upward_queue_size: Default::default(), + max_downward_message_size: Default::default(), + max_upward_message_size: Default::default(), + max_upward_message_num_per_candidate: Default::default(), + hrmp_sender_deposit: Default::default(), + hrmp_recipient_deposit: Default::default(), + hrmp_channel_max_capacity: Default::default(), + hrmp_channel_max_total_size: Default::default(), + hrmp_max_parachain_inbound_channels: Default::default(), + hrmp_channel_max_message_size: Default::default(), + hrmp_max_parachain_outbound_channels: Default::default(), + hrmp_max_message_num_per_candidate: Default::default(), + pvf_voting_ttl: 2u32.into(), + minimum_validation_upgrade_delay: 2.into(), + executor_params: Default::default(), + on_demand_queue_max_size: ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + on_demand_base_fee: 10_000_000u128, + on_demand_fee_variability: Perbill::from_percent(3), + on_demand_target_queue_utilization: Perbill::from_percent(25), + on_demand_ttl: 5u32.into(), + minimum_backing_votes: LEGACY_MIN_BACKING_VOTES, + } + } +} mod v8 { use super::*; @@ -151,9 +254,6 @@ on_demand_fee_variability : Perbill::from_percent(3), on_demand_target_queue_utilization : Perbill::from_percent(25), on_demand_ttl : 5u32.into(), minimum_backing_votes : LEGACY_MIN_BACKING_VOTES, -approval_voting_params : ApprovalVotingParams { - max_approval_coalesce_count: 1, - } } }; diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index 5956b0e155bb..9c84c8840cd5 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -1560,6 +1560,7 @@ pub mod migrations { parachains_configuration::migration::v9::MigrateToV9, // Migrate parachain info format paras_registrar::migration::VersionCheckedMigrateToV1, + parachains_configuration::migration::v10::MigrateToV10, ); } diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 7046c4640c04..83eb3314b49c 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1561,6 +1561,7 @@ pub mod migrations { parachains_configuration::migration::v8::MigrateToV8, parachains_configuration::migration::v9::MigrateToV9, paras_registrar::migration::VersionCheckedMigrateToV1, + parachains_configuration::migration::v10::MigrateToV10, ); } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 9af18b5be2bb..c70d88f766ba 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1445,6 +1445,7 @@ pub mod migrations { UpgradeSessionKeys, parachains_configuration::migration::v9::MigrateToV9, paras_registrar::migration::VersionCheckedMigrateToV1, + parachains_configuration::migration::v10::MigrateToV10, ); } From e70b11326f73ca7fbc0d52ee2d444352a48902c9 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 26 Sep 2023 18:01:03 +0300 Subject: [PATCH 18/89] Fixup failure of lint-markdown E.g: https://github.com/paritytech/polkadot-sdk/actions/runs/6313437255/job/17141490461?pr=1178 Signed-off-by: Alexandru Gheorghe --- .../src/node/approval/approval-voting.md | 26 +++++++++---------- .../src/protocol-approval.md | 26 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index e5b3cfd79751..1a17f90d9ba3 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -267,25 +267,25 @@ entry. The cert itself contains information necessary to determine the candidate * Determine the claimed core index by looking up the candidate with given index in `block_entry.candidates`. Return `AssignmentCheckResult::Bad` if missing. * Check the assignment cert - * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < - session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input - `block_entry.relay_vrf_story ++ sample.encode()` as described with - [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We set - `core_index = vrf.make_bytes().to_u32() % session_info.n_cores`. If the `BlockEntry` causes + * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < + session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input + `block_entry.relay_vrf_story ++ sample.encode()` as described with + [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We set + `core_index = vrf.make_bytes().to_u32() % session_info.n_cores`. If the `BlockEntry` causes inclusion of a candidate at `core_index`, then this is a valid assignment for the candidate at `core_index` and has delay tranche 0. Otherwise, it can be ignored. * If the cert kind is `RelayVRFModuloCompact`, then the certificate is valid as long as the VRF - is valid for the validator's key with the input `block_entry.relay_vrf_story ++ relay_vrf_samples.encode()` - as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). - We enforce that all `core_bitfield` indices are included in the set of the core indices sampled from the + is valid for the validator's key with the input `block_entry.relay_vrf_story ++ relay_vrf_samples.encode()` + as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). + We enforce that all `core_bitfield` indices are included in the set of the core indices sampled from the VRF Output. The assignment is considered a valid tranche0 assignment for all claimed candidates if all `core_bitfield` indices match the core indices where the claimed candidates were included at. - * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the - input `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol - section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not - cause inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included - candidate. The delay tranche for the assignment is determined by reducing + * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the + input `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol + section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not + cause inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included + candidate. The delay tranche for the assignment is determined by reducing `(vrf.make_bytes().to_u64() % (session_info.n_delay_tranches + session_info.zeroth_delay_tranche_width)).saturating_sub(session_info.zeroth_delay_tranche_width)`. * We also check that the core index derived by the output is covered by the `VRFProof` by means of an auxiliary signature. * If the delay tranche is too far in the future, return `AssignmentCheckResult::TooFarInFuture`. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index c614b64410ec..4fac84591456 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -189,9 +189,9 @@ Assignment criteria compute actual assignments using stories and the validators' Assignment criteria output a `Position` consisting of both a `ParaId` to be checked, as well as a precedence `DelayTranche` for when the assignment becomes valid. -Assignment criteria come in four flavors, `RelayVRFModuloCompact`, `RelayVRFDelay`, `RelayEquivocation` and the -deprecated `RelayVRFModulo`. Among these, `RelayVRFModulo`, `RelayVRFModuloCompact` and `RelayVRFDelay` run a -VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the +Assignment criteria come in four flavors, `RelayVRFModuloCompact`, `RelayVRFDelay`, `RelayEquivocation` and the +deprecated `RelayVRFModulo`. Among these, `RelayVRFModulo`, `RelayVRFModuloCompact` and `RelayVRFDelay` run a +VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. Among these, we have two distinct VRF output computations: @@ -204,14 +204,14 @@ sampled availability core in this relay chain block. We choose three samples in secure and efficient by increasing this to four or five, and reducing the backing checks accordingly. All successful `RelayVRFModulo` samples are assigned delay tranche zero. -`RelayVRFModuloCompact` runs a single samples whose VRF input is the `RelayVRFStory` and the sample count. Similar -to `RelayVRFModulo` introduces multiple core assignments for tranche zero. It computes the VRF output with -`schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core v2" and samples up to 160 bytes of the output -as an array of `u32`. Then reduces each `u32` modulo the number of availability cores, and outputs up +`RelayVRFModuloCompact` runs a single samples whose VRF input is the `RelayVRFStory` and the sample count. Similar +to `RelayVRFModulo` introduces multiple core assignments for tranche zero. It computes the VRF output with +`schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core v2" and samples up to 160 bytes of the output +as an array of `u32`. Then reduces each `u32` modulo the number of availability cores, and outputs up to `relay_vrf_modulo_samples` availability core indices. There is no sampling process for `RelayVRFDelay` and `RelayEquivocation`. We instead run them on specific candidates -and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared +and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared available by, a relay chain block, and inputs the associated VRF output via `RelayVRFStory`. `RelayEquivocation` runs only on candidate block equivocations, and inputs their block hashes via the `RelayEquivocation` story. @@ -229,14 +229,14 @@ cannot merge those with the same delay and different stories because `RelayEquiv We track all validators' announced approval assignments for each candidate associated to each relay chain block, which tells us which validators were assigned to which candidates. -We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both -the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo/RelayVRFModuloCompact` +We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both +the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo/RelayVRFModuloCompact` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. -We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the +We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary -because early announcements gives an adversary information. All delay tranche zero assignments always get announced, +because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` and `RelayVRFModuloCompact` assignments. In other words, if some candidate `C` needs more approval checkers by the time we reach round `t` then any validators @@ -325,7 +325,7 @@ finality. We might explore limits on postponement too, but this sounds much har ## Parameters -We prefer doing approval checkers assignments under `RelayVRFModulo` or `RelayVRFModuloCompact` as opposed to +We prefer doing approval checkers assignments under `RelayVRFModulo` or `RelayVRFModuloCompact` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` or `RelayVRFModuloCompact` although assignment levels have never been properly analyzed. From 85939bb864ebd68638fbaa9e120a99e7a417733d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 26 Sep 2023 18:26:04 +0300 Subject: [PATCH 19/89] Fixup test-rustdoc job ... failure example here: https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/3799036 Signed-off-by: Alexandru Gheorghe --- polkadot/node/primitives/src/approval.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polkadot/node/primitives/src/approval.rs b/polkadot/node/primitives/src/approval.rs index bfa1f1aa47d2..e5ae24f7a51e 100644 --- a/polkadot/node/primitives/src/approval.rs +++ b/polkadot/node/primitives/src/approval.rs @@ -365,7 +365,7 @@ pub mod v2 { /// Multiple assignment stories based on the VRF that authorized the relay-chain block /// where the candidates were included. /// - /// The context is [`v2::RELAY_VRF_MODULO_CONTEXT`] + /// The context is [`super::v2::RELAY_VRF_MODULO_CONTEXT`] #[codec(index = 0)] RelayVRFModuloCompact { /// A bitfield representing the core indices claimed by this assignment. @@ -374,7 +374,7 @@ pub mod v2 { /// An assignment story based on the VRF that authorized the relay-chain block where the /// candidate was included combined with the index of a particular core. /// - /// The context is [`v2::RELAY_VRF_DELAY_CONTEXT`] + /// The context is [`super::v1::RELAY_VRF_DELAY_CONTEXT`] #[codec(index = 1)] RelayVRFDelay { /// The core index chosen in this cert. @@ -384,7 +384,7 @@ pub mod v2 { /// An assignment story based on the VRF that authorized the relay-chain block where the /// candidate was included combined with a sample number. /// - /// The context used to produce bytes is [`v1::RELAY_VRF_MODULO_CONTEXT`] + /// The context used to produce bytes is [`super::v1::RELAY_VRF_MODULO_CONTEXT`] #[codec(index = 2)] RelayVRFModulo { /// The sample number used in this cert. From d42f3734e2344cf117f2ec5126f9729fbb04a947 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 27 Sep 2023 09:55:19 +0300 Subject: [PATCH 20/89] Cosmetic fixes Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/availability-recovery/src/lib.rs | 1 - polkadot/primitives/src/runtime_api.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index a040b0fe0aaa..e2146981da92 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -78,7 +78,6 @@ const LRU_SIZE: u32 = 16; const COST_INVALID_REQUEST: Rep = Rep::CostMajor("Peer sent unparsable request"); /// PoV size limit in bytes for which prefer fetching from backers. -//TODO: Cleanup increased for versi testing const SMALL_POV_LIMIT: usize = 128 * 1024; #[derive(Clone, PartialEq)] diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index efa6246250bb..17191d2f7722 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -241,7 +241,7 @@ sp_api::decl_runtime_apis! { key_ownership_proof: vstaging::slashing::OpaqueKeyOwnershipProof, ) -> Option<()>; - /***** Staging *****/ + /***** Staging *****/ /// Get the minimum number of backing votes for a parachain candidate. /// This is a staging method! Do not use on production runtimes! From 6dd51735462615a328580e7f9244b1b76a062e32 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 27 Sep 2023 10:26:50 +0300 Subject: [PATCH 21/89] pipeline/zombinet: fix 0006 test name Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/zombienet/polkadot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 8d3b21eeff7a..5914f50c3ede 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -111,7 +111,7 @@ zombienet-polkadot-functional-0006-parachains-max-tranche0: script: - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="${LOCAL_DIR}/functional" - --test="0006-parachains-0006-parachains-max-tranche0.zndsl" + --test="0006-parachains-max-tranche0.zndsl" zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: From 0bdc06e9d0cbf03f35e637d01d3691acabb001f5 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 27 Sep 2023 10:30:58 +0300 Subject: [PATCH 22/89] fixup cargo fmt nightly Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index eaebac187f18..d3c3fa67a186 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -3847,7 +3847,8 @@ async fn handle_approval_on_max_wait_time( .lock() .set_tick(MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick + TICK_NOW_BEGIN); - // Third time we fetch the configuration when timer expires and we are ready to sent the approval + // Third time we fetch the configuration when timer expires and we are ready to sent the + // approval assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { From 218748db342ee6242d0295a62482a05489cdec42 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 27 Sep 2023 10:45:12 +0300 Subject: [PATCH 23/89] fixup clippy Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index d3c3fa67a186..ec7ec7b9bba8 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -3740,7 +3740,7 @@ async fn handle_approval_on_max_coalesce_count( for _ in &candidate_indicies { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -3804,7 +3804,7 @@ async fn handle_approval_on_max_wait_time( for _ in &candidate_indicies { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, _, timeout, tx)) if timeout == PvfExecTimeoutKind::Approval => { tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } From 0335cf1f5adc519829bf6f18a7b9156583cc0cd9 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 28 Sep 2023 11:15:11 +0300 Subject: [PATCH 24/89] Fixup markdownlint Signed-off-by: Alexandru Gheorghe --- .../src/node/approval/approval-voting.md | 19 ++++++++++--------- .../src/protocol-approval.md | 8 ++++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index 40d959b0beb9..8d7daa173abc 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -3,13 +3,13 @@ Reading the [section on the approval protocol](../../protocol-approval.md) will likely be necessary to understand the aims of this subsystem. -Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to -indicate intent to check a candidate. Upon successfully checking, they don't immediately send the vote instead -they queue the check for a short period of time `MAX_APPROVALS_COALESCE_TICKS` to give the opportunity of the +Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to +indicate intent to check a candidate. Upon successfully checking, they don't immediately send the vote instead +they queue the check for a short period of time `MAX_APPROVALS_COALESCE_TICKS` to give the opportunity of the validator to vote for more than one candidate. Once MAX_APPROVALS_COALESCE_TICKS have passed or at least -`MAX_APPROVAL_COALESCE_COUNT` are ready they broadcast an approval vote for all candidates. If a validator -doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are -being prevented from recovering or validating the block data and that more validators should self-select to +`MAX_APPROVAL_COALESCE_COUNT` are ready they broadcast an approval vote for all candidates. If a validator +doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are +being prevented from recovering or validating the block data and that more validators should self-select to check the candidate. This is known as a "no-show". The core of this subsystem is a Tick-based timer loop, where Ticks are 500ms. We also reason about time in terms of @@ -313,9 +313,9 @@ entry. The cert itself contains information necessary to determine the candidate On receiving a `CheckAndImportApproval(indirect_approval_vote, response_channel)` message: * Fetch the `BlockEntry` from the indirect approval vote's `block_hash`. If none, return `ApprovalCheckResult::Bad`. - * Fetch all `CandidateEntry` from the indirect approval vote's `candidate_indices`. If the block did not trigger + * Fetch all `CandidateEntry` from the indirect approval vote's `candidate_indices`. If the block did not trigger inclusion of enough candidates, return `ApprovalCheckResult::Bad`. - * Construct a `SignedApprovalVote` using the candidates hashes and check against the validator's approval key, + * Construct a `SignedApprovalVote` using the candidates hashes and check against the validator's approval key, based on the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. * Send `ApprovalCheckResult::Accepted` * [Import the checked approval vote](#import-checked-approval) for all candidates @@ -422,7 +422,8 @@ On receiving an `ApprovedAncestor(Hash, BlockNumber, response_channel)`: * Arm a per BlockEntry timer with latest tick we can send the vote. ### Delayed vote distribution - * [Issue Approval Vote](#issue-approval-vote) arms once a per block timer if there are no requirements to send the vote immediately. + * [Issue Approval Vote](#issue-approval-vote) arms once a per block timer if there are no requirements to send the + vote immediately. * When the timer wakes up it will either: * IF there is a candidate in the queue past its sending tick: * Construct a `SignedApprovalVote` with the validator index for the session and all candidate hashes in the waiting queue. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index 86e496fb9eba..1a217afb9b71 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -297,10 +297,14 @@ provide somewhat more security. TODO: When? Is this optimal for the network? etc. ## Approval coalescing -To reduce the necessary network bandwidth and cpu time when a validator has more than one candidate to approve we are doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we delay the sending of it untill one of three things happen: +To reduce the necessary network bandwidth and cpu time when a validator has more than one candidate to approve we are +doing our best effort to send a single message that approves all available candidates with a single signature. +The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we +delay the sending of it untill one of three things happen: - We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we are ready to vote for. - `MAX_APPROVALS_COALESCE_TICKS` have passed since the we were ready to approve the candidate. -- We are already in the last third of the now-show period in order to avoid creating accidental no shows, which in turn my trigger other assignments. +- We are already in the last third of the now-show period in order to avoid creating accidental no shows, which in + turn my trigger other assignments. ## On-chain verification From 0f0faf0e39680e503ab333eabe64dcd38bf8e997 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 28 Sep 2023 11:58:57 +0300 Subject: [PATCH 25/89] Fixup runtime-migration-westend Signed-off-by: Alexandru Gheorghe --- polkadot/runtime/parachains/src/configuration.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index 6690e4156c7f..32b3bb665669 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -505,7 +505,8 @@ pub mod pallet { /// v6-v7: /// v7-v8: /// v8-v9: - const STORAGE_VERSION: StorageVersion = StorageVersion::new(9); + /// v9-10: + const STORAGE_VERSION: StorageVersion = StorageVersion::new(10); #[pallet::pallet] #[pallet::storage_version(STORAGE_VERSION)] From 6eabb634a52da1a7003aa63de14d5e661f714bb3 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 28 Sep 2023 16:23:46 +0300 Subject: [PATCH 26/89] approval-voting: address round 1 of feedback Signed-off-by: Alexandru Gheorghe --- .../approval-voting/src/approval_checking.rs | 64 ++++---- .../src/approval_db/v2/migration_helpers.rs | 6 +- .../approval-voting/src/approval_db/v2/mod.rs | 10 +- .../node/core/approval-voting/src/backend.rs | 3 +- .../node/core/approval-voting/src/criteria.rs | 2 +- polkadot/node/core/approval-voting/src/lib.rs | 56 +++---- .../approval-voting/src/persisted_entries.rs | 146 ++++++++++++------ .../node/core/approval-voting/src/tests.rs | 2 + .../network/approval-distribution/src/lib.rs | 4 +- 9 files changed, 166 insertions(+), 127 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index 2f0ea4160ae1..374b55aa0b4f 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -467,13 +467,15 @@ mod tests { #[test] fn pending_is_not_approved() { - let candidate = approval_db::v1::CandidateEntry { - candidate: dummy_candidate_receipt(dummy_hash()), - session: 0, - block_assignments: BTreeMap::default(), - approvals: BitVec::default(), - } - .into(); + let candidate = CandidateEntry::from_v1( + approval_db::v1::CandidateEntry { + candidate: dummy_candidate_receipt(dummy_hash()), + session: 0, + block_assignments: BTreeMap::default(), + approvals: BitVec::default(), + }, + 0, + ); let approval_entry = approval_db::v2::ApprovalEntry { tranches: Vec::new(), @@ -500,13 +502,15 @@ mod tests { #[test] fn exact_takes_only_assignments_up_to() { - let mut candidate: CandidateEntry = approval_db::v1::CandidateEntry { - candidate: dummy_candidate_receipt(dummy_hash()), - session: 0, - block_assignments: BTreeMap::default(), - approvals: bitvec![u8, BitOrderLsb0; 0; 10], - } - .into(); + let mut candidate: CandidateEntry = CandidateEntry::from_v1( + approval_db::v1::CandidateEntry { + candidate: dummy_candidate_receipt(dummy_hash()), + session: 0, + block_assignments: BTreeMap::default(), + approvals: bitvec![u8, BitOrderLsb0; 0; 10], + }, + 0, + ); for i in 0..3 { candidate.mark_approval(ValidatorIndex(i)); @@ -572,13 +576,15 @@ mod tests { #[test] fn one_honest_node_always_approves() { - let mut candidate: CandidateEntry = approval_db::v1::CandidateEntry { - candidate: dummy_candidate_receipt(dummy_hash()), - session: 0, - block_assignments: BTreeMap::default(), - approvals: bitvec![u8, BitOrderLsb0; 0; 10], - } - .into(); + let mut candidate: CandidateEntry = CandidateEntry::from_v1( + approval_db::v1::CandidateEntry { + candidate: dummy_candidate_receipt(dummy_hash()), + session: 0, + block_assignments: BTreeMap::default(), + approvals: bitvec![u8, BitOrderLsb0; 0; 10], + }, + 0, + ); for i in 0..3 { candidate.mark_approval(ValidatorIndex(i)); @@ -1043,13 +1049,15 @@ mod tests { let no_show_duration = 10; let needed_approvals = 3; - let mut candidate: CandidateEntry = approval_db::v1::CandidateEntry { - candidate: dummy_candidate_receipt(dummy_hash()), - session: 0, - block_assignments: BTreeMap::default(), - approvals: bitvec![u8, BitOrderLsb0; 0; 3], - } - .into(); + let mut candidate: CandidateEntry = CandidateEntry::from_v1( + approval_db::v1::CandidateEntry { + candidate: dummy_candidate_receipt(dummy_hash()), + session: 0, + block_assignments: BTreeMap::default(), + approvals: bitvec![u8, BitOrderLsb0; 0; 3], + }, + 0, + ); for i in 0..3 { candidate.mark_approval(ValidatorIndex(i)); diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs index a70d766fc8c9..54efa3dc8efc 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -86,11 +86,13 @@ pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { let mut counter = 0; // Get all candidate entries, approval entries and convert each of them. for block in all_blocks { - for (_core_index, candidate_hash) in block.candidates() { + for (candidate_index, (_core_index, candidate_hash)) in + block.candidates().iter().enumerate() + { // Loading the candidate will also perform the conversion to the updated format and // return that represantation. if let Some(candidate_entry) = backend - .load_candidate_entry_v1(&candidate_hash) + .load_candidate_entry_v1(&candidate_hash, candidate_index as CandidateIndex) .map_err(|e| Error::InternalError(e))? { // Write the updated representation. diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs index 88650a539130..2846b0ca499b 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -62,9 +62,10 @@ impl V1ReadBackend for DbBackend { fn load_candidate_entry_v1( &self, candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, ) -> SubsystemResult> { load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) - .map(|e| e.map(Into::into)) + .map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v1(e, candidate_index))) } fn load_block_entry_v1( @@ -205,9 +206,8 @@ pub struct TrancheEntry { pub struct OurApproval { /// The signature for the candidates hashes pointed by indices. pub signature: ValidatorSignature, - /// The indices of the candidates signed in this approval, an empty value means only - /// the candidate referred by this approval entry was signed. - pub signed_candidates_indices: Option, + /// The indices of the candidates signed in this approval. + pub signed_candidates_indices: CandidateBitfield, } /// Metadata regarding approval of a particular candidate within the context of some @@ -270,7 +270,7 @@ pub struct CandidateSigningContext { /// The candidate hash, to be included in the signature. pub candidate_hash: CandidateHash, /// The latest tick we have to create and send the approval. - pub send_no_later_than_tick: Tick, + pub sign_no_later_than_tick: Tick, } impl From for Tick { diff --git a/polkadot/node/core/approval-voting/src/backend.rs b/polkadot/node/core/approval-voting/src/backend.rs index 374e7a826d19..37d9c76d252f 100644 --- a/polkadot/node/core/approval-voting/src/backend.rs +++ b/polkadot/node/core/approval-voting/src/backend.rs @@ -22,7 +22,7 @@ //! before any commit to the underlying storage is made. use polkadot_node_subsystem::SubsystemResult; -use polkadot_primitives::{BlockNumber, CandidateHash, Hash}; +use polkadot_primitives::{BlockNumber, CandidateHash, CandidateIndex, Hash}; use std::collections::HashMap; @@ -72,6 +72,7 @@ pub trait V1ReadBackend: Backend { fn load_candidate_entry_v1( &self, candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, ) -> SubsystemResult>; /// Load a block entry from the DB with scheme version 1. diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index db74302f0225..1f751e2bf140 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -278,7 +278,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) } fn check_assignment_cert( diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 95d04cc0d147..cb46782d4878 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -1346,7 +1346,7 @@ fn distribution_messages_for_activation( session: block_entry.session(), }); let mut signatures_queued = HashSet::new(); - for (i, (_, candidate_hash)) in block_entry.candidates().iter().enumerate() { + for (_, candidate_hash) in block_entry.candidates() { let _candidate_span = distribution_message_span.child("candidate").with_candidate(*candidate_hash); let candidate_entry = match db.load_candidate_entry(&candidate_hash)? { @@ -1449,14 +1449,14 @@ fn distribution_messages_for_activation( continue }, } - let candidate_indices = approval_sig - .signed_candidates_indices - .unwrap_or((i as CandidateIndex).into()); - if signatures_queued.insert(candidate_indices.clone()) { + if signatures_queued + .insert(approval_sig.signed_candidates_indices.clone()) + { messages.push(ApprovalDistributionMessage::DistributeApproval( IndirectSignedApprovalVoteV2 { block_hash, - candidate_indices, + candidate_indices: approval_sig + .signed_candidates_indices, validator: assignment.validator_index(), signature: approval_sig.signature, }, @@ -3384,23 +3384,15 @@ async fn maybe_create_signature( target: LOG_TARGET, "Candidates pending signatures {:}", block_entry.num_candidates_pending_signature() ); - - let oldest_candidate_to_sign = match block_entry.longest_waiting_candidate_signature() { - Some(candidate) => candidate, - // No cached candidates, nothing to do here, this just means the timer fired, - // but the signatures were already sent because we gathered more than - // max_approval_coalesce_count. - None => return Ok(None), - }; - let tick_now = state.clock.tick_now(); - if oldest_candidate_to_sign.send_no_later_than_tick > tick_now && - (block_entry.num_candidates_pending_signature() as u32) < - approval_params.max_approval_coalesce_count - { - return Ok(Some(oldest_candidate_to_sign.send_no_later_than_tick)) - } + let (candidates_to_sign, sign_no_later_then) = block_entry + .get_candidates_that_need_signature(tick_now, approval_params.max_approval_coalesce_count); + + let (candidates_hashes, candidates_indices) = match candidates_to_sign { + Some(candidates_to_sign) => candidates_to_sign, + None => return Ok(sign_no_later_then), + }; let session_info = match get_session_info( session_info_provider, @@ -3436,12 +3428,10 @@ async fn maybe_create_signature( }, }; - let candidate_hashes = block_entry.candidate_hashes_pending_signature(); - let signature = match sign_approval( &state.keystore, &validator_pubkey, - candidate_hashes.clone(), + candidates_hashes.clone(), block_entry.session(), ) { Some(sig) => sig, @@ -3457,14 +3447,13 @@ async fn maybe_create_signature( return Ok(None) }, }; + metrics.on_approval_coalesce(candidates_hashes.len() as u32); - let candidate_entries = candidate_hashes + let candidate_entries = candidates_hashes .iter() .map(|candidate_hash| db.load_candidate_entry(candidate_hash)) .collect::>>>()?; - let candidate_indices = block_entry.candidate_indices_pending_signature(); - for candidate_entry in candidate_entries { let mut candidate_entry = candidate_entry .expect("Candidate was scheduled to be signed entry in db should exist; qed"); @@ -3473,26 +3462,17 @@ async fn maybe_create_signature( .expect("Candidate was scheduled to be signed entry in db should exist; qed"); approval_entry.import_approval_sig(OurApproval { signature: signature.clone(), - signed_candidates_indices: Some( - candidate_indices - .clone() - .try_into() - .expect("Fails only of array empty, it can't be, qed"), - ), + signed_candidates_indices: candidates_indices.clone(), }); db.write_candidate_entry(candidate_entry); } - metrics.on_approval_coalesce(candidate_indices.len() as u32); - metrics.on_approval_produced(); ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeApproval( IndirectSignedApprovalVoteV2 { block_hash: block_entry.block_hash(), - candidate_indices: candidate_indices - .try_into() - .expect("Fails only of array empty, it can't be, qed"), + candidate_indices: candidates_indices, validator: validator_index, signature, }, diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index e441c23be122..d01d3c8fe9d8 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -20,6 +20,7 @@ //! Within that context, things are plain-old-data. Within this module, //! data and logic are intertwined. +use itertools::Itertools; use polkadot_node_primitives::approval::{ v1::{DelayTranche, RelayVRFStory}, v2::{AssignmentCertV2, CandidateBitfield}, @@ -93,22 +94,22 @@ impl From for crate::approval_db::v2::OurApproval { } } -impl From for OurApproval { - fn from(value: ValidatorSignature) -> Self { - Self { signature: value, signed_candidates_indices: Default::default() } - } -} - /// Metadata about our approval signature #[derive(Debug, Clone, PartialEq)] pub struct OurApproval { /// The signature for the candidates hashes pointed by indices. pub signature: ValidatorSignature, - /// The indices of the candidates signed in this approval, an empty value means only - /// the candidate referred by this approval entry was signed. - pub signed_candidates_indices: Option, + /// The indices of the candidates signed in this approval. + pub signed_candidates_indices: CandidateBitfield, } +impl OurApproval { + /// Converts a ValidatorSignature to an OurApproval. + /// It used in converting the database from v1 to v2. + pub fn from_v1(value: ValidatorSignature, candidate_index: CandidateIndex) -> Self { + Self { signature: value, signed_candidates_indices: candidate_index.into() } + } +} /// Metadata regarding approval of a particular candidate within the context of some /// particular block. #[derive(Debug, Clone, PartialEq)] @@ -265,6 +266,23 @@ impl ApprovalEntry { (None, None) } } + + // Convert an ApprovalEntry from v1 version to a v2 version + pub fn from_v1( + value: crate::approval_db::v1::ApprovalEntry, + candidate_index: CandidateIndex, + ) -> Self { + ApprovalEntry { + tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), + backing_group: value.backing_group, + our_assignment: value.our_assignment.map(|assignment| assignment.into()), + our_approval_sig: value + .our_approval_sig + .map(|sig| OurApproval::from_v1(sig, candidate_index)), + assigned_validators: value.assignments, + approved: value.approved, + } + } } impl From for ApprovalEntry { @@ -336,6 +354,23 @@ impl CandidateEntry { pub fn approval_entry(&self, block_hash: &Hash) -> Option<&ApprovalEntry> { self.block_assignments.get(block_hash) } + + /// Convert a CandidateEntry from a v1 to its v2 equivalent. + pub fn from_v1( + value: crate::approval_db::v1::CandidateEntry, + candidate_index: CandidateIndex, + ) -> Self { + Self { + approvals: value.approvals, + block_assignments: value + .block_assignments + .into_iter() + .map(|(h, ae)| (h, ApprovalEntry::from_v1(ae, candidate_index))) + .collect(), + candidate: value.candidate, + session: value.session, + } + } } impl From for CandidateEntry { @@ -398,7 +433,7 @@ pub struct BlockEntry { #[derive(Debug, Clone, PartialEq)] pub struct CandidateSigningContext { pub candidate_hash: CandidateHash, - pub send_no_later_than_tick: Tick, + pub sign_no_later_than_tick: Tick, } impl BlockEntry { @@ -495,11 +530,11 @@ impl BlockEntry { &mut self, candidate_index: CandidateIndex, candidate_hash: CandidateHash, - send_no_later_than_tick: Tick, + sign_no_later_than_tick: Tick, ) -> Option { self.candidates_pending_signature.insert( candidate_index, - CandidateSigningContext { candidate_hash, send_no_later_than_tick }, + CandidateSigningContext { candidate_hash, sign_no_later_than_tick }, ) } @@ -514,7 +549,7 @@ impl BlockEntry { } /// Candidate hashes for candidates pending signatures - pub fn candidate_hashes_pending_signature(&self) -> Vec { + fn candidate_hashes_pending_signature(&self) -> Vec { self.candidates_pending_signature .values() .map(|unsigned_approval| unsigned_approval.candidate_hash) @@ -522,15 +557,55 @@ impl BlockEntry { } /// Candidate indices for candidates pending signature - pub fn candidate_indices_pending_signature(&self) -> Vec { - self.candidates_pending_signature.keys().map(|val| *val).collect() + fn candidate_indices_pending_signature(&self) -> CandidateBitfield { + self.candidates_pending_signature + .keys() + .map(|val| *val) + .collect_vec() + .try_into() + .expect("Fails only of array empty, it can't be, qed") } - /// Returns the candidate that has been longest in the queue. - pub fn longest_waiting_candidate_signature(&self) -> Option<&CandidateSigningContext> { - self.candidates_pending_signature + /// Returns a list of candidates hashes that need need signature created at the current tick: + /// This might happen in other of the two reasons: + /// 1. We queued more than max_approval_coalesce_count candidates. + /// 2. We have candidates that waiting in the queue past their `sign_no_later_than_tick` + /// + /// Additionally, we also return the first tick when we will have to create a signature, + /// so that the caller can arm the timer if it is not already armed. + pub fn get_candidates_that_need_signature( + &self, + tick_now: Tick, + max_approval_coalesce_count: u32, + ) -> (Option<(Vec, CandidateBitfield)>, Option) { + let sign_no_later_than_tick = self + .candidates_pending_signature .values() - .min_by(|a, b| a.send_no_later_than_tick.cmp(&b.send_no_later_than_tick)) + .min_by(|a, b| a.sign_no_later_than_tick.cmp(&b.sign_no_later_than_tick)) + .map(|val| val.sign_no_later_than_tick); + + if let Some(sign_no_later_than_tick) = sign_no_later_than_tick { + if sign_no_later_than_tick <= tick_now || + self.num_candidates_pending_signature() >= max_approval_coalesce_count as usize + { + ( + Some(( + self.candidate_hashes_pending_signature(), + self.candidate_indices_pending_signature(), + )), + Some(sign_no_later_than_tick), + ) + } else { + // We can still wait for other candidates to queue in, so just make sure + // we wake up at the tick we have to sign the longest waiting candidate. + (Default::default(), Some(sign_no_later_than_tick)) + } + } else { + // No cached candidates, nothing to do here, this just means the timer fired, + // but the signatures were already sent because we gathered more than + // max_approval_coalesce_count. + (Default::default(), sign_no_later_than_tick) + } } /// Signals the approval was issued for the candidates pending signature @@ -605,7 +680,7 @@ impl From for CandidateSigningC fn from(signing_context: crate::approval_db::v2::CandidateSigningContext) -> Self { Self { candidate_hash: signing_context.candidate_hash, - send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), + sign_no_later_than_tick: signing_context.sign_no_later_than_tick.into(), } } } @@ -614,36 +689,7 @@ impl From for crate::approval_db::v2::CandidateSigningC fn from(signing_context: CandidateSigningContext) -> Self { Self { candidate_hash: signing_context.candidate_hash, - send_no_later_than_tick: signing_context.send_no_later_than_tick.into(), - } - } -} - -/// Migration helpers. -impl From for CandidateEntry { - fn from(value: crate::approval_db::v1::CandidateEntry) -> Self { - Self { - approvals: value.approvals, - block_assignments: value - .block_assignments - .into_iter() - .map(|(h, ae)| (h, ae.into())) - .collect(), - candidate: value.candidate, - session: value.session, - } - } -} - -impl From for ApprovalEntry { - fn from(value: crate::approval_db::v1::ApprovalEntry) -> Self { - ApprovalEntry { - tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), - backing_group: value.backing_group, - our_assignment: value.our_assignment.map(|assignment| assignment.into()), - our_approval_sig: value.our_approval_sig.map(Into::into), - assigned_validators: value.assignments, - approved: value.approved, + sign_no_later_than_tick: signing_context.sign_no_later_than_tick.into(), } } } diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index ec7ec7b9bba8..04aef90aae97 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -280,6 +280,7 @@ impl V1ReadBackend for TestStoreInner { fn load_candidate_entry_v1( &self, candidate_hash: &CandidateHash, + _candidate_index: CandidateIndex, ) -> SubsystemResult> { self.load_candidate_entry(candidate_hash) } @@ -363,6 +364,7 @@ impl V1ReadBackend for TestStore { fn load_candidate_entry_v1( &self, candidate_hash: &CandidateHash, + _candidate_index: CandidateIndex, ) -> SubsystemResult> { self.load_candidate_entry(candidate_hash) } diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index d63732bbe04b..8162128755a6 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -134,7 +134,7 @@ enum ApprovalEntryError { CandidateIndexOutOfBounds, InvalidCandidateIndex, DuplicateApproval, - CouldNotFindAssignment, + UnknownAssignment, AssignmentsFollowedDifferentPaths(RequiredRouting, RequiredRouting), } @@ -580,7 +580,7 @@ impl BlockEntry { if let Some(required_routing) = required_routing { Ok(required_routing) } else { - Err(ApprovalEntryError::CouldNotFindAssignment) + Err(ApprovalEntryError::UnknownAssignment) } } From 9dc4a93441898d7d7f3031fc788406da322beecd Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 28 Sep 2023 18:29:50 +0300 Subject: [PATCH 27/89] approval-distribution: cosmetic improvements Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 8162128755a6..7411b9c7b545 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -180,6 +180,16 @@ impl ApprovalEntry { self.routing_info.required_routing = required_routing; } + // Tells if this entry assignment covers at least one candidate in the approval + pub fn includes_approval_candidates(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { + for candidate_index in approval.candidate_indices.iter_ones() { + if self.assignment_claimed_candidates.bit_at((candidate_index).as_bit_index()) { + return true + } + } + return false + } + // Records a new approval. Returns false if the claimed candidate is not found or we already // have received the approval. pub fn note_approval( @@ -195,13 +205,7 @@ impl ApprovalEntry { } // We need at least one of the candidates in the approval to be in this assignment - if !approval - .candidate_indices - .iter_ones() - .fold(false, |accumulator, candidate_index| { - self.assignment_claimed_candidates.bit_at((candidate_index).as_bit_index()) || - accumulator - }) { + if !self.includes_approval_candidates(&approval) { return Err(ApprovalEntryError::InvalidCandidateIndex) } @@ -405,6 +409,13 @@ impl Knowledge { } success } + + // Tells if all keys are contained by this peer_knowledge + pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { + keys.iter().fold(true, |accumulator, assignment_key| { + accumulator && self.contains(&assignment_key.0, assignment_key.1) + }) + } } /// Information that has been circulated to and from a peer. @@ -423,12 +434,12 @@ impl PeerKnowledge { // Tells if all assignments for a given approval are included in the knowledge of the peer fn contains_assignments_for_approval(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { - Self::generate_assignments_keys(&approval).iter().fold( - true, - |accumulator, assignment_key| { - accumulator && self.contains(&assignment_key.0, assignment_key.1) - }, - ) + self.contains_all_keys(&Self::generate_assignments_keys(&approval)) + } + + // Tells if all keys are contained by this peer_knowledge + pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { + self.sent.contains_all_keys(keys) || self.received.contains_all_keys(keys) } // Generate the knowledge keys for querying if all assignments of an approval are known @@ -584,6 +595,7 @@ impl BlockEntry { } } + /// Returns the list of approval votes covering this candidate pub fn approval_votes( &self, candidate_index: CandidateIndex, @@ -1723,10 +1735,7 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || - assignments_knowledge_keys.iter().fold(true, |result, message_subject| { - result && knowledge.sent.contains(&message_subject.0, message_subject.1) - }) + in_topology || knowledge.sent.contains_all_keys(&assignments_knowledge_keys) }; let peers = entry From 566d7f5c439a356fedae415691d08315ade22735 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 29 Sep 2023 12:39:10 +0300 Subject: [PATCH 28/89] Comment vstaging Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/protocol/src/lib.rs | 516 +++++++++++----------- 1 file changed, 258 insertions(+), 258 deletions(-) diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index ba9b9a7f4900..5d8b3cd550ca 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -588,264 +588,264 @@ pub mod v1 { } } -/// vstaging network protocol types. -pub mod vstaging { - use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; - use parity_scale_codec::{Decode, Encode}; - - use polkadot_primitives::vstaging::{ - CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, - UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, - }; - - use polkadot_node_primitives::{ - approval::{ - v1::IndirectSignedApprovalVote, - v2::{CandidateBitfield, IndirectAssignmentCertV2}, - }, - UncheckedSignedFullStatement, - }; - - /// Network messages used by the bitfield distribution subsystem. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub enum BitfieldDistributionMessage { - /// A signed availability bitfield for a given relay-parent hash. - #[codec(index = 0)] - Bitfield(Hash, UncheckedSignedAvailabilityBitfield), - } - - /// Bitfields indicating the statements that are known or undesired - /// about a candidate. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub struct StatementFilter { - /// Seconded statements. '1' is known or undesired. - pub seconded_in_group: BitVec, - /// Valid statements. '1' is known or undesired. - pub validated_in_group: BitVec, - } - - impl StatementFilter { - /// Create a new blank filter with the given group size. - pub fn blank(group_size: usize) -> Self { - StatementFilter { - seconded_in_group: BitVec::repeat(false, group_size), - validated_in_group: BitVec::repeat(false, group_size), - } - } - - /// Create a new full filter with the given group size. - pub fn full(group_size: usize) -> Self { - StatementFilter { - seconded_in_group: BitVec::repeat(true, group_size), - validated_in_group: BitVec::repeat(true, group_size), - } - } - - /// Whether the filter has a specific expected length, consistent across both - /// bitfields. - pub fn has_len(&self, len: usize) -> bool { - self.seconded_in_group.len() == len && self.validated_in_group.len() == len - } - - /// Determine the number of backing validators in the statement filter. - pub fn backing_validators(&self) -> usize { - self.seconded_in_group - .iter() - .by_vals() - .zip(self.validated_in_group.iter().by_vals()) - .filter(|&(s, v)| s || v) // no double-counting - .count() - } - - /// Whether the statement filter has at least one seconded statement. - pub fn has_seconded(&self) -> bool { - self.seconded_in_group.iter().by_vals().any(|x| x) - } - - /// Mask out `Seconded` statements in `self` according to the provided - /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. - pub fn mask_seconded(&mut self, mask: &BitSlice) { - for (mut x, mask) in self - .seconded_in_group - .iter_mut() - .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) - { - // (x, mask) => x - // (true, true) => false - // (true, false) => true - // (false, true) => false - // (false, false) => false - *x = *x && !mask; - } - } - - /// Mask out `Valid` statements in `self` according to the provided - /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. - pub fn mask_valid(&mut self, mask: &BitSlice) { - for (mut x, mask) in self - .validated_in_group - .iter_mut() - .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) - { - // (x, mask) => x - // (true, true) => false - // (true, false) => true - // (false, true) => false - // (false, false) => false - *x = *x && !mask; - } - } - } - - /// A manifest of a known backed candidate, along with a description - /// of the statements backing it. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub struct BackedCandidateManifest { - /// The relay-parent of the candidate. - pub relay_parent: Hash, - /// The hash of the candidate. - pub candidate_hash: CandidateHash, - /// The group index backing the candidate at the relay-parent. - pub group_index: GroupIndex, - /// The para ID of the candidate. It is illegal for this to - /// be a para ID which is not assigned to the group indicated - /// in this manifest. - pub para_id: ParaId, - /// The head-data corresponding to the candidate. - pub parent_head_data_hash: Hash, - /// A statement filter which indicates which validators in the - /// para's group at the relay-parent have validated this candidate - /// and issued statements about it, to the advertiser's knowledge. - /// - /// This MUST have exactly the minimum amount of bytes - /// necessary to represent the number of validators in the assigned - /// backing group as-of the relay-parent. - pub statement_knowledge: StatementFilter, - } - - /// An acknowledgement of a backed candidate being known. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub struct BackedCandidateAcknowledgement { - /// The hash of the candidate. - pub candidate_hash: CandidateHash, - /// A statement filter which indicates which validators in the - /// para's group at the relay-parent have validated this candidate - /// and issued statements about it, to the advertiser's knowledge. - /// - /// This MUST have exactly the minimum amount of bytes - /// necessary to represent the number of validators in the assigned - /// backing group as-of the relay-parent. - pub statement_knowledge: StatementFilter, - } - - /// Network messages used by the statement distribution subsystem. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub enum StatementDistributionMessage { - /// A notification of a signed statement in compact form, for a given relay parent. - #[codec(index = 0)] - Statement(Hash, UncheckedSignedStatement), - - /// A notification of a backed candidate being known by the - /// sending node, for the purpose of being requested by the receiving node - /// if needed. - #[codec(index = 1)] - BackedCandidateManifest(BackedCandidateManifest), - - /// A notification of a backed candidate being known by the sending node, - /// for the purpose of informing a receiving node which already has the candidate. - #[codec(index = 2)] - BackedCandidateKnown(BackedCandidateAcknowledgement), - - /// All messages for V1 for compatibility with the statement distribution - /// protocol, for relay-parents that don't support asynchronous backing. - /// - /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents - /// which support asynchronous backing. This backwards compatibility should be - /// considered immediately deprecated and can be removed once the node software - /// is not required to support logic from before asynchronous backing anymore. - #[codec(index = 255)] - V1Compatibility(crate::v1::StatementDistributionMessage), - } - - /// Network messages used by the approval distribution subsystem. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub enum ApprovalDistributionMessage { - /// Assignments for candidates in recent, unfinalized blocks. - /// We use a bitfield to reference claimed candidates, where the bit index is equal to - /// candidate index. - /// - /// Actually checking the assignment may yield a different result. - /// TODO: Look at getting rid of bitfield in the future. - #[codec(index = 0)] - Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), - /// Approvals for candidates in some recent, unfinalized block. - #[codec(index = 1)] - Approvals(Vec), - } - - /// Dummy network message type, so we will receive connect/disconnect events. - #[derive(Debug, Clone, PartialEq, Eq)] - pub enum GossipSupportNetworkMessage {} - - /// Network messages used by the collator protocol subsystem - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] - pub enum CollatorProtocolMessage { - /// Declare the intent to advertise collations under a collator ID, attaching a - /// signature of the `PeerId` of the node using the given collator ID key. - #[codec(index = 0)] - Declare(CollatorId, ParaId, CollatorSignature), - /// Advertise a collation to a validator. Can only be sent once the peer has - /// declared that they are a collator with given ID. - #[codec(index = 1)] - AdvertiseCollation { - /// Hash of the relay parent advertised collation is based on. - relay_parent: Hash, - /// Candidate hash. - candidate_hash: CandidateHash, - /// Parachain head data hash before candidate execution. - parent_head_data_hash: Hash, - }, - /// A collation sent to a validator was seconded. - #[codec(index = 4)] - CollationSeconded(Hash, UncheckedSignedFullStatement), - } - - /// All network messages on the validation peer-set. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] - pub enum ValidationProtocol { - /// Bitfield distribution messages - #[codec(index = 1)] - #[from] - BitfieldDistribution(BitfieldDistributionMessage), - /// Statement distribution messages - #[codec(index = 3)] - #[from] - StatementDistribution(StatementDistributionMessage), - /// Approval distribution messages - #[codec(index = 4)] - #[from] - ApprovalDistribution(ApprovalDistributionMessage), - } - - /// All network messages on the collation peer-set. - #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] - pub enum CollationProtocol { - /// Collator protocol messages - #[codec(index = 0)] - #[from] - CollatorProtocol(CollatorProtocolMessage), - } - - /// Get the payload that should be signed and included in a `Declare` message. - /// - /// The payload is the local peer id of the node, which serves to prove that it - /// controls the collator key it is declaring an intention to collate under. - pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { - let mut payload = peer_id.to_bytes(); - payload.extend_from_slice(b"COLL"); - payload - } -} +// /// vstaging network protocol types. +// pub mod vstaging { +// use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; +// use parity_scale_codec::{Decode, Encode}; + +// use polkadot_primitives::vstaging::{ +// CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, +// UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, +// }; + +// use polkadot_node_primitives::{ +// approval::{ +// v1::IndirectSignedApprovalVote, +// v2::{CandidateBitfield, IndirectAssignmentCertV2}, +// }, +// UncheckedSignedFullStatement, +// }; + +// /// Network messages used by the bitfield distribution subsystem. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub enum BitfieldDistributionMessage { +// /// A signed availability bitfield for a given relay-parent hash. +// #[codec(index = 0)] +// Bitfield(Hash, UncheckedSignedAvailabilityBitfield), +// } + +// /// Bitfields indicating the statements that are known or undesired +// /// about a candidate. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub struct StatementFilter { +// /// Seconded statements. '1' is known or undesired. +// pub seconded_in_group: BitVec, +// /// Valid statements. '1' is known or undesired. +// pub validated_in_group: BitVec, +// } + +// impl StatementFilter { +// /// Create a new blank filter with the given group size. +// pub fn blank(group_size: usize) -> Self { +// StatementFilter { +// seconded_in_group: BitVec::repeat(false, group_size), +// validated_in_group: BitVec::repeat(false, group_size), +// } +// } + +// /// Create a new full filter with the given group size. +// pub fn full(group_size: usize) -> Self { +// StatementFilter { +// seconded_in_group: BitVec::repeat(true, group_size), +// validated_in_group: BitVec::repeat(true, group_size), +// } +// } + +// /// Whether the filter has a specific expected length, consistent across both +// /// bitfields. +// pub fn has_len(&self, len: usize) -> bool { +// self.seconded_in_group.len() == len && self.validated_in_group.len() == len +// } + +// /// Determine the number of backing validators in the statement filter. +// pub fn backing_validators(&self) -> usize { +// self.seconded_in_group +// .iter() +// .by_vals() +// .zip(self.validated_in_group.iter().by_vals()) +// .filter(|&(s, v)| s || v) // no double-counting +// .count() +// } + +// /// Whether the statement filter has at least one seconded statement. +// pub fn has_seconded(&self) -> bool { +// self.seconded_in_group.iter().by_vals().any(|x| x) +// } + +// /// Mask out `Seconded` statements in `self` according to the provided +// /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. +// pub fn mask_seconded(&mut self, mask: &BitSlice) { +// for (mut x, mask) in self +// .seconded_in_group +// .iter_mut() +// .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) +// { +// // (x, mask) => x +// // (true, true) => false +// // (true, false) => true +// // (false, true) => false +// // (false, false) => false +// *x = *x && !mask; +// } +// } + +// /// Mask out `Valid` statements in `self` according to the provided +// /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. +// pub fn mask_valid(&mut self, mask: &BitSlice) { +// for (mut x, mask) in self +// .validated_in_group +// .iter_mut() +// .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) +// { +// // (x, mask) => x +// // (true, true) => false +// // (true, false) => true +// // (false, true) => false +// // (false, false) => false +// *x = *x && !mask; +// } +// } +// } + +// /// A manifest of a known backed candidate, along with a description +// /// of the statements backing it. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub struct BackedCandidateManifest { +// /// The relay-parent of the candidate. +// pub relay_parent: Hash, +// /// The hash of the candidate. +// pub candidate_hash: CandidateHash, +// /// The group index backing the candidate at the relay-parent. +// pub group_index: GroupIndex, +// /// The para ID of the candidate. It is illegal for this to +// /// be a para ID which is not assigned to the group indicated +// /// in this manifest. +// pub para_id: ParaId, +// /// The head-data corresponding to the candidate. +// pub parent_head_data_hash: Hash, +// /// A statement filter which indicates which validators in the +// /// para's group at the relay-parent have validated this candidate +// /// and issued statements about it, to the advertiser's knowledge. +// /// +// /// This MUST have exactly the minimum amount of bytes +// /// necessary to represent the number of validators in the assigned +// /// backing group as-of the relay-parent. +// pub statement_knowledge: StatementFilter, +// } + +// /// An acknowledgement of a backed candidate being known. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub struct BackedCandidateAcknowledgement { +// /// The hash of the candidate. +// pub candidate_hash: CandidateHash, +// /// A statement filter which indicates which validators in the +// /// para's group at the relay-parent have validated this candidate +// /// and issued statements about it, to the advertiser's knowledge. +// /// +// /// This MUST have exactly the minimum amount of bytes +// /// necessary to represent the number of validators in the assigned +// /// backing group as-of the relay-parent. +// pub statement_knowledge: StatementFilter, +// } + +// /// Network messages used by the statement distribution subsystem. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub enum StatementDistributionMessage { +// /// A notification of a signed statement in compact form, for a given relay parent. +// #[codec(index = 0)] +// Statement(Hash, UncheckedSignedStatement), + +// /// A notification of a backed candidate being known by the +// /// sending node, for the purpose of being requested by the receiving node +// /// if needed. +// #[codec(index = 1)] +// BackedCandidateManifest(BackedCandidateManifest), + +// /// A notification of a backed candidate being known by the sending node, +// /// for the purpose of informing a receiving node which already has the candidate. +// #[codec(index = 2)] +// BackedCandidateKnown(BackedCandidateAcknowledgement), + +// /// All messages for V1 for compatibility with the statement distribution +// /// protocol, for relay-parents that don't support asynchronous backing. +// /// +// /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents +// /// which support asynchronous backing. This backwards compatibility should be +// /// considered immediately deprecated and can be removed once the node software +// /// is not required to support logic from before asynchronous backing anymore. +// #[codec(index = 255)] +// V1Compatibility(crate::v1::StatementDistributionMessage), +// } + +// /// Network messages used by the approval distribution subsystem. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub enum ApprovalDistributionMessage { +// /// Assignments for candidates in recent, unfinalized blocks. +// /// We use a bitfield to reference claimed candidates, where the bit index is equal to +// /// candidate index. +// /// +// /// Actually checking the assignment may yield a different result. +// /// TODO: Look at getting rid of bitfield in the future. +// #[codec(index = 0)] +// Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), +// /// Approvals for candidates in some recent, unfinalized block. +// #[codec(index = 1)] +// Approvals(Vec), +// } + +// /// Dummy network message type, so we will receive connect/disconnect events. +// #[derive(Debug, Clone, PartialEq, Eq)] +// pub enum GossipSupportNetworkMessage {} + +// /// Network messages used by the collator protocol subsystem +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] +// pub enum CollatorProtocolMessage { +// /// Declare the intent to advertise collations under a collator ID, attaching a +// /// signature of the `PeerId` of the node using the given collator ID key. +// #[codec(index = 0)] +// Declare(CollatorId, ParaId, CollatorSignature), +// /// Advertise a collation to a validator. Can only be sent once the peer has +// /// declared that they are a collator with given ID. +// #[codec(index = 1)] +// AdvertiseCollation { +// /// Hash of the relay parent advertised collation is based on. +// relay_parent: Hash, +// /// Candidate hash. +// candidate_hash: CandidateHash, +// /// Parachain head data hash before candidate execution. +// parent_head_data_hash: Hash, +// }, +// /// A collation sent to a validator was seconded. +// #[codec(index = 4)] +// CollationSeconded(Hash, UncheckedSignedFullStatement), +// } + +// /// All network messages on the validation peer-set. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] +// pub enum ValidationProtocol { +// /// Bitfield distribution messages +// #[codec(index = 1)] +// #[from] +// BitfieldDistribution(BitfieldDistributionMessage), +// /// Statement distribution messages +// #[codec(index = 3)] +// #[from] +// StatementDistribution(StatementDistributionMessage), +// /// Approval distribution messages +// #[codec(index = 4)] +// #[from] +// ApprovalDistribution(ApprovalDistributionMessage), +// } + +// /// All network messages on the collation peer-set. +// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] +// pub enum CollationProtocol { +// /// Collator protocol messages +// #[codec(index = 0)] +// #[from] +// CollatorProtocol(CollatorProtocolMessage), +// } + +// /// Get the payload that should be signed and included in a `Declare` message. +// /// +// /// The payload is the local peer id of the node, which serves to prove that it +// /// controls the collator key it is declaring an intention to collate under. +// pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { +// let mut payload = peer_id.to_bytes(); +// payload.extend_from_slice(b"COLL"); +// payload +// } +// } /// Returns the subset of `peers` with the specified `version`. pub fn filter_by_peer_version( From 460bab2508c9b8bc2e1c06d1091581857e8d4982 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 29 Sep 2023 12:43:58 +0300 Subject: [PATCH 29/89] Revert "Comment vstaging" This reverts commit 566d7f5c439a356fedae415691d08315ade22735. --- polkadot/node/network/protocol/src/lib.rs | 516 +++++++++++----------- 1 file changed, 258 insertions(+), 258 deletions(-) diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index 5d8b3cd550ca..ba9b9a7f4900 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -588,264 +588,264 @@ pub mod v1 { } } -// /// vstaging network protocol types. -// pub mod vstaging { -// use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; -// use parity_scale_codec::{Decode, Encode}; - -// use polkadot_primitives::vstaging::{ -// CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, -// UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, -// }; - -// use polkadot_node_primitives::{ -// approval::{ -// v1::IndirectSignedApprovalVote, -// v2::{CandidateBitfield, IndirectAssignmentCertV2}, -// }, -// UncheckedSignedFullStatement, -// }; - -// /// Network messages used by the bitfield distribution subsystem. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub enum BitfieldDistributionMessage { -// /// A signed availability bitfield for a given relay-parent hash. -// #[codec(index = 0)] -// Bitfield(Hash, UncheckedSignedAvailabilityBitfield), -// } - -// /// Bitfields indicating the statements that are known or undesired -// /// about a candidate. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub struct StatementFilter { -// /// Seconded statements. '1' is known or undesired. -// pub seconded_in_group: BitVec, -// /// Valid statements. '1' is known or undesired. -// pub validated_in_group: BitVec, -// } - -// impl StatementFilter { -// /// Create a new blank filter with the given group size. -// pub fn blank(group_size: usize) -> Self { -// StatementFilter { -// seconded_in_group: BitVec::repeat(false, group_size), -// validated_in_group: BitVec::repeat(false, group_size), -// } -// } - -// /// Create a new full filter with the given group size. -// pub fn full(group_size: usize) -> Self { -// StatementFilter { -// seconded_in_group: BitVec::repeat(true, group_size), -// validated_in_group: BitVec::repeat(true, group_size), -// } -// } - -// /// Whether the filter has a specific expected length, consistent across both -// /// bitfields. -// pub fn has_len(&self, len: usize) -> bool { -// self.seconded_in_group.len() == len && self.validated_in_group.len() == len -// } - -// /// Determine the number of backing validators in the statement filter. -// pub fn backing_validators(&self) -> usize { -// self.seconded_in_group -// .iter() -// .by_vals() -// .zip(self.validated_in_group.iter().by_vals()) -// .filter(|&(s, v)| s || v) // no double-counting -// .count() -// } - -// /// Whether the statement filter has at least one seconded statement. -// pub fn has_seconded(&self) -> bool { -// self.seconded_in_group.iter().by_vals().any(|x| x) -// } - -// /// Mask out `Seconded` statements in `self` according to the provided -// /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. -// pub fn mask_seconded(&mut self, mask: &BitSlice) { -// for (mut x, mask) in self -// .seconded_in_group -// .iter_mut() -// .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) -// { -// // (x, mask) => x -// // (true, true) => false -// // (true, false) => true -// // (false, true) => false -// // (false, false) => false -// *x = *x && !mask; -// } -// } - -// /// Mask out `Valid` statements in `self` according to the provided -// /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. -// pub fn mask_valid(&mut self, mask: &BitSlice) { -// for (mut x, mask) in self -// .validated_in_group -// .iter_mut() -// .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) -// { -// // (x, mask) => x -// // (true, true) => false -// // (true, false) => true -// // (false, true) => false -// // (false, false) => false -// *x = *x && !mask; -// } -// } -// } - -// /// A manifest of a known backed candidate, along with a description -// /// of the statements backing it. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub struct BackedCandidateManifest { -// /// The relay-parent of the candidate. -// pub relay_parent: Hash, -// /// The hash of the candidate. -// pub candidate_hash: CandidateHash, -// /// The group index backing the candidate at the relay-parent. -// pub group_index: GroupIndex, -// /// The para ID of the candidate. It is illegal for this to -// /// be a para ID which is not assigned to the group indicated -// /// in this manifest. -// pub para_id: ParaId, -// /// The head-data corresponding to the candidate. -// pub parent_head_data_hash: Hash, -// /// A statement filter which indicates which validators in the -// /// para's group at the relay-parent have validated this candidate -// /// and issued statements about it, to the advertiser's knowledge. -// /// -// /// This MUST have exactly the minimum amount of bytes -// /// necessary to represent the number of validators in the assigned -// /// backing group as-of the relay-parent. -// pub statement_knowledge: StatementFilter, -// } - -// /// An acknowledgement of a backed candidate being known. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub struct BackedCandidateAcknowledgement { -// /// The hash of the candidate. -// pub candidate_hash: CandidateHash, -// /// A statement filter which indicates which validators in the -// /// para's group at the relay-parent have validated this candidate -// /// and issued statements about it, to the advertiser's knowledge. -// /// -// /// This MUST have exactly the minimum amount of bytes -// /// necessary to represent the number of validators in the assigned -// /// backing group as-of the relay-parent. -// pub statement_knowledge: StatementFilter, -// } - -// /// Network messages used by the statement distribution subsystem. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub enum StatementDistributionMessage { -// /// A notification of a signed statement in compact form, for a given relay parent. -// #[codec(index = 0)] -// Statement(Hash, UncheckedSignedStatement), - -// /// A notification of a backed candidate being known by the -// /// sending node, for the purpose of being requested by the receiving node -// /// if needed. -// #[codec(index = 1)] -// BackedCandidateManifest(BackedCandidateManifest), - -// /// A notification of a backed candidate being known by the sending node, -// /// for the purpose of informing a receiving node which already has the candidate. -// #[codec(index = 2)] -// BackedCandidateKnown(BackedCandidateAcknowledgement), - -// /// All messages for V1 for compatibility with the statement distribution -// /// protocol, for relay-parents that don't support asynchronous backing. -// /// -// /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents -// /// which support asynchronous backing. This backwards compatibility should be -// /// considered immediately deprecated and can be removed once the node software -// /// is not required to support logic from before asynchronous backing anymore. -// #[codec(index = 255)] -// V1Compatibility(crate::v1::StatementDistributionMessage), -// } - -// /// Network messages used by the approval distribution subsystem. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub enum ApprovalDistributionMessage { -// /// Assignments for candidates in recent, unfinalized blocks. -// /// We use a bitfield to reference claimed candidates, where the bit index is equal to -// /// candidate index. -// /// -// /// Actually checking the assignment may yield a different result. -// /// TODO: Look at getting rid of bitfield in the future. -// #[codec(index = 0)] -// Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), -// /// Approvals for candidates in some recent, unfinalized block. -// #[codec(index = 1)] -// Approvals(Vec), -// } - -// /// Dummy network message type, so we will receive connect/disconnect events. -// #[derive(Debug, Clone, PartialEq, Eq)] -// pub enum GossipSupportNetworkMessage {} - -// /// Network messages used by the collator protocol subsystem -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] -// pub enum CollatorProtocolMessage { -// /// Declare the intent to advertise collations under a collator ID, attaching a -// /// signature of the `PeerId` of the node using the given collator ID key. -// #[codec(index = 0)] -// Declare(CollatorId, ParaId, CollatorSignature), -// /// Advertise a collation to a validator. Can only be sent once the peer has -// /// declared that they are a collator with given ID. -// #[codec(index = 1)] -// AdvertiseCollation { -// /// Hash of the relay parent advertised collation is based on. -// relay_parent: Hash, -// /// Candidate hash. -// candidate_hash: CandidateHash, -// /// Parachain head data hash before candidate execution. -// parent_head_data_hash: Hash, -// }, -// /// A collation sent to a validator was seconded. -// #[codec(index = 4)] -// CollationSeconded(Hash, UncheckedSignedFullStatement), -// } - -// /// All network messages on the validation peer-set. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] -// pub enum ValidationProtocol { -// /// Bitfield distribution messages -// #[codec(index = 1)] -// #[from] -// BitfieldDistribution(BitfieldDistributionMessage), -// /// Statement distribution messages -// #[codec(index = 3)] -// #[from] -// StatementDistribution(StatementDistributionMessage), -// /// Approval distribution messages -// #[codec(index = 4)] -// #[from] -// ApprovalDistribution(ApprovalDistributionMessage), -// } - -// /// All network messages on the collation peer-set. -// #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] -// pub enum CollationProtocol { -// /// Collator protocol messages -// #[codec(index = 0)] -// #[from] -// CollatorProtocol(CollatorProtocolMessage), -// } - -// /// Get the payload that should be signed and included in a `Declare` message. -// /// -// /// The payload is the local peer id of the node, which serves to prove that it -// /// controls the collator key it is declaring an intention to collate under. -// pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { -// let mut payload = peer_id.to_bytes(); -// payload.extend_from_slice(b"COLL"); -// payload -// } -// } +/// vstaging network protocol types. +pub mod vstaging { + use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; + use parity_scale_codec::{Decode, Encode}; + + use polkadot_primitives::vstaging::{ + CandidateHash, CollatorId, CollatorSignature, GroupIndex, Hash, Id as ParaId, + UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, + }; + + use polkadot_node_primitives::{ + approval::{ + v1::IndirectSignedApprovalVote, + v2::{CandidateBitfield, IndirectAssignmentCertV2}, + }, + UncheckedSignedFullStatement, + }; + + /// Network messages used by the bitfield distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum BitfieldDistributionMessage { + /// A signed availability bitfield for a given relay-parent hash. + #[codec(index = 0)] + Bitfield(Hash, UncheckedSignedAvailabilityBitfield), + } + + /// Bitfields indicating the statements that are known or undesired + /// about a candidate. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct StatementFilter { + /// Seconded statements. '1' is known or undesired. + pub seconded_in_group: BitVec, + /// Valid statements. '1' is known or undesired. + pub validated_in_group: BitVec, + } + + impl StatementFilter { + /// Create a new blank filter with the given group size. + pub fn blank(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(false, group_size), + validated_in_group: BitVec::repeat(false, group_size), + } + } + + /// Create a new full filter with the given group size. + pub fn full(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(true, group_size), + validated_in_group: BitVec::repeat(true, group_size), + } + } + + /// Whether the filter has a specific expected length, consistent across both + /// bitfields. + pub fn has_len(&self, len: usize) -> bool { + self.seconded_in_group.len() == len && self.validated_in_group.len() == len + } + + /// Determine the number of backing validators in the statement filter. + pub fn backing_validators(&self) -> usize { + self.seconded_in_group + .iter() + .by_vals() + .zip(self.validated_in_group.iter().by_vals()) + .filter(|&(s, v)| s || v) // no double-counting + .count() + } + + /// Whether the statement filter has at least one seconded statement. + pub fn has_seconded(&self) -> bool { + self.seconded_in_group.iter().by_vals().any(|x| x) + } + + /// Mask out `Seconded` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_seconded(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .seconded_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + + /// Mask out `Valid` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_valid(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .validated_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + } + + /// A manifest of a known backed candidate, along with a description + /// of the statements backing it. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateManifest { + /// The relay-parent of the candidate. + pub relay_parent: Hash, + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// The group index backing the candidate at the relay-parent. + pub group_index: GroupIndex, + /// The para ID of the candidate. It is illegal for this to + /// be a para ID which is not assigned to the group indicated + /// in this manifest. + pub para_id: ParaId, + /// The head-data corresponding to the candidate. + pub parent_head_data_hash: Hash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + + /// An acknowledgement of a backed candidate being known. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateAcknowledgement { + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + + /// Network messages used by the statement distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum StatementDistributionMessage { + /// A notification of a signed statement in compact form, for a given relay parent. + #[codec(index = 0)] + Statement(Hash, UncheckedSignedStatement), + + /// A notification of a backed candidate being known by the + /// sending node, for the purpose of being requested by the receiving node + /// if needed. + #[codec(index = 1)] + BackedCandidateManifest(BackedCandidateManifest), + + /// A notification of a backed candidate being known by the sending node, + /// for the purpose of informing a receiving node which already has the candidate. + #[codec(index = 2)] + BackedCandidateKnown(BackedCandidateAcknowledgement), + + /// All messages for V1 for compatibility with the statement distribution + /// protocol, for relay-parents that don't support asynchronous backing. + /// + /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents + /// which support asynchronous backing. This backwards compatibility should be + /// considered immediately deprecated and can be removed once the node software + /// is not required to support logic from before asynchronous backing anymore. + #[codec(index = 255)] + V1Compatibility(crate::v1::StatementDistributionMessage), + } + + /// Network messages used by the approval distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum ApprovalDistributionMessage { + /// Assignments for candidates in recent, unfinalized blocks. + /// We use a bitfield to reference claimed candidates, where the bit index is equal to + /// candidate index. + /// + /// Actually checking the assignment may yield a different result. + /// TODO: Look at getting rid of bitfield in the future. + #[codec(index = 0)] + Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>), + /// Approvals for candidates in some recent, unfinalized block. + #[codec(index = 1)] + Approvals(Vec), + } + + /// Dummy network message type, so we will receive connect/disconnect events. + #[derive(Debug, Clone, PartialEq, Eq)] + pub enum GossipSupportNetworkMessage {} + + /// Network messages used by the collator protocol subsystem + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum CollatorProtocolMessage { + /// Declare the intent to advertise collations under a collator ID, attaching a + /// signature of the `PeerId` of the node using the given collator ID key. + #[codec(index = 0)] + Declare(CollatorId, ParaId, CollatorSignature), + /// Advertise a collation to a validator. Can only be sent once the peer has + /// declared that they are a collator with given ID. + #[codec(index = 1)] + AdvertiseCollation { + /// Hash of the relay parent advertised collation is based on. + relay_parent: Hash, + /// Candidate hash. + candidate_hash: CandidateHash, + /// Parachain head data hash before candidate execution. + parent_head_data_hash: Hash, + }, + /// A collation sent to a validator was seconded. + #[codec(index = 4)] + CollationSeconded(Hash, UncheckedSignedFullStatement), + } + + /// All network messages on the validation peer-set. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] + pub enum ValidationProtocol { + /// Bitfield distribution messages + #[codec(index = 1)] + #[from] + BitfieldDistribution(BitfieldDistributionMessage), + /// Statement distribution messages + #[codec(index = 3)] + #[from] + StatementDistribution(StatementDistributionMessage), + /// Approval distribution messages + #[codec(index = 4)] + #[from] + ApprovalDistribution(ApprovalDistributionMessage), + } + + /// All network messages on the collation peer-set. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] + pub enum CollationProtocol { + /// Collator protocol messages + #[codec(index = 0)] + #[from] + CollatorProtocol(CollatorProtocolMessage), + } + + /// Get the payload that should be signed and included in a `Declare` message. + /// + /// The payload is the local peer id of the node, which serves to prove that it + /// controls the collator key it is declaring an intention to collate under. + pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { + let mut payload = peer_id.to_bytes(); + payload.extend_from_slice(b"COLL"); + payload + } +} /// Returns the subset of `peers` with the specified `version`. pub fn filter_by_peer_version( From 85acb511da241c80c27ea47c823ed69ad3d0b2f2 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 29 Sep 2023 12:58:59 +0300 Subject: [PATCH 30/89] Add v3 protocol Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/protocol/src/lib.rs | 309 ++++++++++++++++++++-- 1 file changed, 294 insertions(+), 15 deletions(-) diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index ba9b9a7f4900..a17a483b97de 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -253,18 +253,21 @@ impl View { /// A protocol-versioned type. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Versioned { +pub enum Versioned { /// V1 type. V1(V1), + /// V2 type. + V2(V2), /// VStaging type. VStaging(VStaging), } -impl Versioned<&'_ V1, &'_ VStaging> { +impl Versioned<&'_ V1, &'_ V2, &'_ VStaging> { /// Convert to a fully-owned version of the message. - pub fn clone_inner(&self) -> Versioned { + pub fn clone_inner(&self) -> Versioned { match *self { Versioned::V1(inner) => Versioned::V1(inner.clone()), + Versioned::V2(inner) => Versioned::V2(inner.clone()), Versioned::VStaging(inner) => Versioned::VStaging(inner.clone()), } } @@ -272,7 +275,7 @@ impl Versioned<&'_ V1, &'_ VStaging> { /// All supported versions of the validation protocol message. pub type VersionedValidationProtocol = - Versioned; + Versioned; impl From for VersionedValidationProtocol { fn from(v1: v1::ValidationProtocol) -> Self { @@ -287,7 +290,8 @@ impl From for VersionedValidationProtocol { } /// All supported versions of the collation protocol message. -pub type VersionedCollationProtocol = Versioned; +pub type VersionedCollationProtocol = + Versioned; impl From for VersionedCollationProtocol { fn from(v1: v1::CollationProtocol) -> Self { @@ -307,6 +311,7 @@ macro_rules! impl_versioned_full_protocol_from { fn from(versioned_from: $from) -> $out { match versioned_from { Versioned::V1(x) => Versioned::V1(x.into()), + Versioned::V2(x) => Versioned::V2(x.into()), Versioned::VStaging(x) => Versioned::VStaging(x.into()), } } @@ -320,6 +325,7 @@ macro_rules! impl_versioned_try_from { $from:ty, $out:ty, $v1_pat:pat => $v1_out:expr, + $v2_pat:pat => $v2_out:expr, $vstaging_pat:pat => $vstaging_out:expr ) => { impl TryFrom<$from> for $out { @@ -329,6 +335,7 @@ macro_rules! impl_versioned_try_from { #[allow(unreachable_patterns)] // when there is only one variant match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out)), + Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out)), Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out)), _ => Err(crate::WrongVariant), } @@ -352,8 +359,11 @@ macro_rules! impl_versioned_try_from { } /// Version-annotated messages used by the bitfield distribution subsystem. -pub type BitfieldDistributionMessage = - Versioned; +pub type BitfieldDistributionMessage = Versioned< + v1::BitfieldDistributionMessage, + v2::BitfieldDistributionMessage, + vstaging::BitfieldDistributionMessage, +>; impl_versioned_full_protocol_from!( BitfieldDistributionMessage, VersionedValidationProtocol, @@ -363,12 +373,16 @@ impl_versioned_try_from!( VersionedValidationProtocol, BitfieldDistributionMessage, v1::ValidationProtocol::BitfieldDistribution(x) => x, + v2::ValidationProtocol::BitfieldDistribution(x) => x, vstaging::ValidationProtocol::BitfieldDistribution(x) => x ); /// Version-annotated messages used by the statement distribution subsystem. -pub type StatementDistributionMessage = - Versioned; +pub type StatementDistributionMessage = Versioned< + v1::StatementDistributionMessage, + v2::StatementDistributionMessage, + vstaging::StatementDistributionMessage, +>; impl_versioned_full_protocol_from!( StatementDistributionMessage, VersionedValidationProtocol, @@ -378,12 +392,16 @@ impl_versioned_try_from!( VersionedValidationProtocol, StatementDistributionMessage, v1::ValidationProtocol::StatementDistribution(x) => x, + v2::ValidationProtocol::StatementDistribution(x) => x, vstaging::ValidationProtocol::StatementDistribution(x) => x ); /// Version-annotated messages used by the approval distribution subsystem. -pub type ApprovalDistributionMessage = - Versioned; +pub type ApprovalDistributionMessage = Versioned< + v1::ApprovalDistributionMessage, + v2::ApprovalDistributionMessage, + vstaging::ApprovalDistributionMessage, +>; impl_versioned_full_protocol_from!( ApprovalDistributionMessage, VersionedValidationProtocol, @@ -393,13 +411,17 @@ impl_versioned_try_from!( VersionedValidationProtocol, ApprovalDistributionMessage, v1::ValidationProtocol::ApprovalDistribution(x) => x, + v2::ValidationProtocol::ApprovalDistribution(x) => x, vstaging::ValidationProtocol::ApprovalDistribution(x) => x ); /// Version-annotated messages used by the gossip-support subsystem (this is void). -pub type GossipSupportNetworkMessage = - Versioned; +pub type GossipSupportNetworkMessage = Versioned< + v1::GossipSupportNetworkMessage, + v2::GossipSupportNetworkMessage, + vstaging::GossipSupportNetworkMessage, +>; // This is a void enum placeholder, so never gets sent over the wire. impl TryFrom for GossipSupportNetworkMessage { type Error = WrongVariant; @@ -416,8 +438,11 @@ impl<'a> TryFrom<&'a VersionedValidationProtocol> for GossipSupportNetworkMessag } /// Version-annotated messages used by the bitfield distribution subsystem. -pub type CollatorProtocolMessage = - Versioned; +pub type CollatorProtocolMessage = Versioned< + v1::CollatorProtocolMessage, + v2::CollatorProtocolMessage, + vstaging::CollatorProtocolMessage, +>; impl_versioned_full_protocol_from!( CollatorProtocolMessage, VersionedCollationProtocol, @@ -427,6 +452,7 @@ impl_versioned_try_from!( VersionedCollationProtocol, CollatorProtocolMessage, v1::CollationProtocol::CollatorProtocol(x) => x, + v2::CollationProtocol::CollatorProtocol(x) => x, vstaging::CollationProtocol::CollatorProtocol(x) => x ); @@ -588,6 +614,259 @@ pub mod v1 { } } +/// v2 network protocol types. +pub mod v2 { + use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; + use parity_scale_codec::{Decode, Encode}; + + use polkadot_primitives::{ + CandidateHash, CandidateIndex, CollatorId, CollatorSignature, GroupIndex, Hash, + Id as ParaId, UncheckedSignedAvailabilityBitfield, UncheckedSignedStatement, + }; + + use polkadot_node_primitives::{ + approval::{IndirectAssignmentCert, IndirectSignedApprovalVote}, + UncheckedSignedFullStatement, + }; + + /// Network messages used by the bitfield distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum BitfieldDistributionMessage { + /// A signed availability bitfield for a given relay-parent hash. + #[codec(index = 0)] + Bitfield(Hash, UncheckedSignedAvailabilityBitfield), + } + + /// Bitfields indicating the statements that are known or undesired + /// about a candidate. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct StatementFilter { + /// Seconded statements. '1' is known or undesired. + pub seconded_in_group: BitVec, + /// Valid statements. '1' is known or undesired. + pub validated_in_group: BitVec, + } + + impl StatementFilter { + /// Create a new blank filter with the given group size. + pub fn blank(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(false, group_size), + validated_in_group: BitVec::repeat(false, group_size), + } + } + + /// Create a new full filter with the given group size. + pub fn full(group_size: usize) -> Self { + StatementFilter { + seconded_in_group: BitVec::repeat(true, group_size), + validated_in_group: BitVec::repeat(true, group_size), + } + } + + /// Whether the filter has a specific expected length, consistent across both + /// bitfields. + pub fn has_len(&self, len: usize) -> bool { + self.seconded_in_group.len() == len && self.validated_in_group.len() == len + } + + /// Determine the number of backing validators in the statement filter. + pub fn backing_validators(&self) -> usize { + self.seconded_in_group + .iter() + .by_vals() + .zip(self.validated_in_group.iter().by_vals()) + .filter(|&(s, v)| s || v) // no double-counting + .count() + } + + /// Whether the statement filter has at least one seconded statement. + pub fn has_seconded(&self) -> bool { + self.seconded_in_group.iter().by_vals().any(|x| x) + } + + /// Mask out `Seconded` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_seconded(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .seconded_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + + /// Mask out `Valid` statements in `self` according to the provided + /// bitvec. Bits appearing in `mask` will not appear in `self` afterwards. + pub fn mask_valid(&mut self, mask: &BitSlice) { + for (mut x, mask) in self + .validated_in_group + .iter_mut() + .zip(mask.iter().by_vals().chain(std::iter::repeat(false))) + { + // (x, mask) => x + // (true, true) => false + // (true, false) => true + // (false, true) => false + // (false, false) => false + *x = *x && !mask; + } + } + } + + /// A manifest of a known backed candidate, along with a description + /// of the statements backing it. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateManifest { + /// The relay-parent of the candidate. + pub relay_parent: Hash, + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// The group index backing the candidate at the relay-parent. + pub group_index: GroupIndex, + /// The para ID of the candidate. It is illegal for this to + /// be a para ID which is not assigned to the group indicated + /// in this manifest. + pub para_id: ParaId, + /// The head-data corresponding to the candidate. + pub parent_head_data_hash: Hash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + + /// An acknowledgement of a backed candidate being known. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub struct BackedCandidateAcknowledgement { + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// A statement filter which indicates which validators in the + /// para's group at the relay-parent have validated this candidate + /// and issued statements about it, to the advertiser's knowledge. + /// + /// This MUST have exactly the minimum amount of bytes + /// necessary to represent the number of validators in the assigned + /// backing group as-of the relay-parent. + pub statement_knowledge: StatementFilter, + } + + /// Network messages used by the statement distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum StatementDistributionMessage { + /// A notification of a signed statement in compact form, for a given relay parent. + #[codec(index = 0)] + Statement(Hash, UncheckedSignedStatement), + + /// A notification of a backed candidate being known by the + /// sending node, for the purpose of being requested by the receiving node + /// if needed. + #[codec(index = 1)] + BackedCandidateManifest(BackedCandidateManifest), + + /// A notification of a backed candidate being known by the sending node, + /// for the purpose of informing a receiving node which already has the candidate. + #[codec(index = 2)] + BackedCandidateKnown(BackedCandidateAcknowledgement), + + /// All messages for V1 for compatibility with the statement distribution + /// protocol, for relay-parents that don't support asynchronous backing. + /// + /// These are illegal to send to V1 peers, and illegal to send concerning relay-parents + /// which support asynchronous backing. This backwards compatibility should be + /// considered immediately deprecated and can be removed once the node software + /// is not required to support logic from before asynchronous backing anymore. + #[codec(index = 255)] + V1Compatibility(crate::v1::StatementDistributionMessage), + } + + /// Network messages used by the approval distribution subsystem. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum ApprovalDistributionMessage { + /// Assignments for candidates in recent, unfinalized blocks. + /// + /// Actually checking the assignment may yield a different result. + #[codec(index = 0)] + Assignments(Vec<(IndirectAssignmentCert, CandidateIndex)>), + /// Approvals for candidates in some recent, unfinalized block. + #[codec(index = 1)] + Approvals(Vec), + } + + /// Dummy network message type, so we will receive connect/disconnect events. + #[derive(Debug, Clone, PartialEq, Eq)] + pub enum GossipSupportNetworkMessage {} + + /// Network messages used by the collator protocol subsystem + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] + pub enum CollatorProtocolMessage { + /// Declare the intent to advertise collations under a collator ID, attaching a + /// signature of the `PeerId` of the node using the given collator ID key. + #[codec(index = 0)] + Declare(CollatorId, ParaId, CollatorSignature), + /// Advertise a collation to a validator. Can only be sent once the peer has + /// declared that they are a collator with given ID. + #[codec(index = 1)] + AdvertiseCollation { + /// Hash of the relay parent advertised collation is based on. + relay_parent: Hash, + /// Candidate hash. + candidate_hash: CandidateHash, + /// Parachain head data hash before candidate execution. + parent_head_data_hash: Hash, + }, + /// A collation sent to a validator was seconded. + #[codec(index = 4)] + CollationSeconded(Hash, UncheckedSignedFullStatement), + } + + /// All network messages on the validation peer-set. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] + pub enum ValidationProtocol { + /// Bitfield distribution messages + #[codec(index = 1)] + #[from] + BitfieldDistribution(BitfieldDistributionMessage), + /// Statement distribution messages + #[codec(index = 3)] + #[from] + StatementDistribution(StatementDistributionMessage), + /// Approval distribution messages + #[codec(index = 4)] + #[from] + ApprovalDistribution(ApprovalDistributionMessage), + } + + /// All network messages on the collation peer-set. + #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, derive_more::From)] + pub enum CollationProtocol { + /// Collator protocol messages + #[codec(index = 0)] + #[from] + CollatorProtocol(CollatorProtocolMessage), + } + + /// Get the payload that should be signed and included in a `Declare` message. + /// + /// The payload is the local peer id of the node, which serves to prove that it + /// controls the collator key it is declaring an intention to collate under. + pub fn declare_signature_payload(peer_id: &sc_network::PeerId) -> Vec { + let mut payload = peer_id.to_bytes(); + payload.extend_from_slice(b"COLL"); + payload + } +} + /// vstaging network protocol types. pub mod vstaging { use bitvec::{order::Lsb0, slice::BitSlice, vec::BitVec}; From 56055c1c37dab805a2f6c6dd68b01d9570dfa1ba Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sat, 30 Sep 2023 17:09:40 +0300 Subject: [PATCH 31/89] Fixup Signed-off-by: Alexandru Gheorghe --- polkadot/Cargo.toml | 2 + .../network/bitfield-distribution/src/lib.rs | 40 ++- polkadot/node/network/bridge/src/network.rs | 2 +- polkadot/node/network/bridge/src/rx/mod.rs | 59 +++- .../src/collator_side/mod.rs | 24 +- .../src/validator_side/mod.rs | 3 +- polkadot/node/network/protocol/src/lib.rs | 7 + .../node/network/protocol/src/peer_set.rs | 6 +- .../src/legacy_v1/mod.rs | 47 ++- .../statement-distribution/src/v2/mod.rs | 271 +++++++++++++----- .../undying/collator/Cargo.toml | 3 + .../0006-parachains-max-tranche0.toml | 2 +- 12 files changed, 361 insertions(+), 105 deletions(-) diff --git a/polkadot/Cargo.toml b/polkadot/Cargo.toml index 6e82cb69f6ec..06ac13f1bf87 100644 --- a/polkadot/Cargo.toml +++ b/polkadot/Cargo.toml @@ -64,6 +64,8 @@ jemalloc-allocator = [ "polkadot-node-core-pvf/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator", ] +network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] + # Enables timeout-based tests supposed to be run only in CI environment as they may be flaky # when run locally depending on system load diff --git a/polkadot/node/network/bitfield-distribution/src/lib.rs b/polkadot/node/network/bitfield-distribution/src/lib.rs index 7a1a32770a2a..9cc79aee8490 100644 --- a/polkadot/node/network/bitfield-distribution/src/lib.rs +++ b/polkadot/node/network/bitfield-distribution/src/lib.rs @@ -25,14 +25,15 @@ use always_assert::never; use futures::{channel::oneshot, FutureExt}; +use net_protocol::filter_by_peer_version; use polkadot_node_network_protocol::{ self as net_protocol, grid_topology::{ GridNeighbors, RandomRouting, RequiredRouting, SessionBoundGridTopologyStorage, }, peer_set::{ProtocolVersion, ValidationVersion}, - v1 as protocol_v1, v2 as protocol_v2, OurView, PeerId, UnifiedReputationChange as Rep, - Versioned, View, + v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, + UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_subsystem::{ jaeger, messages::*, overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, PerLeafSpan, @@ -96,11 +97,16 @@ impl BitfieldGossipMessage { self.relay_parent, self.signed_availability.into(), )), - Some(ValidationVersion::V2) | Some(ValidationVersion::VStaging) => + Some(ValidationVersion::V2) => Versioned::V2(protocol_v2::BitfieldDistributionMessage::Bitfield( self.relay_parent, self.signed_availability.into(), )), + Some(ValidationVersion::VStaging) => + Versioned::VStaging(protocol_vstaging::BitfieldDistributionMessage::Bitfield( + self.relay_parent, + self.signed_availability.into(), + )), None => { never!("Peers should only have supported protocol versions."); @@ -492,17 +498,13 @@ async fn relay_message( } else { let _span = span.child("gossip"); - let filter_by_version = |peers: &[(PeerId, ProtocolVersion)], - version: ValidationVersion| { - peers - .iter() - .filter(|(_, v)| v == &version.into()) - .map(|(peer_id, _)| *peer_id) - .collect::>() - }; + let v1_interested_peers = + filter_by_peer_version(&interested_peers, ValidationVersion::V1.into()); + let v2_interested_peers = + filter_by_peer_version(&interested_peers, ValidationVersion::V2.into()); - let v1_interested_peers = filter_by_version(&interested_peers, ValidationVersion::V1); - let v2_interested_peers = filter_by_version(&interested_peers, ValidationVersion::V2); + let vstaging_interested_peers = + filter_by_peer_version(&interested_peers, ValidationVersion::VStaging.into()); if !v1_interested_peers.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( @@ -515,7 +517,15 @@ async fn relay_message( if !v2_interested_peers.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( v2_interested_peers, - message.into_validation_protocol(ValidationVersion::V2.into()), + message.clone().into_validation_protocol(ValidationVersion::V2.into()), + )) + .await + } + + if !vstaging_interested_peers.is_empty() { + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + vstaging_interested_peers, + message.into_validation_protocol(ValidationVersion::VStaging.into()), )) .await } @@ -541,7 +551,7 @@ async fn process_incoming_peer_message( relay_parent, bitfield, )) | - Versioned::VStaging(protocol_v2::BitfieldDistributionMessage::Bitfield( + Versioned::VStaging(protocol_vstaging::BitfieldDistributionMessage::Bitfield( relay_parent, bitfield, )) => (relay_parent, bitfield), diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index 3cf6741d4293..a13045285522 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -72,7 +72,7 @@ pub(crate) fn send_validation_message_vstaging( message: WireMessage, metrics: &Metrics, ) { - gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v2 message to peers",); + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation vstaging message to peers",); send_message( net, diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 0d32ece88e28..83bf10bbb670 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -21,6 +21,7 @@ use super::*; use always_assert::never; use bytes::Bytes; use futures::stream::{BoxStream, StreamExt}; +use net_protocol::filter_by_peer_version; use parity_scale_codec::{Decode, DecodeAll}; use sc_network::Event as NetworkEvent; @@ -323,7 +324,7 @@ where &mut network_service, vec![peer], &peerset_protocol_names, - WireMessage::::ViewUpdate( + WireMessage::::ViewUpdate( local_view, ), &metrics, @@ -495,6 +496,16 @@ where v_messages, &metrics, ) + } else if expected_versions[PeerSet::Validation] == + Some(ValidationVersion::VStaging.into()) + { + handle_peer_messages::( + remote, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + v_messages, + &metrics, + ) } else { gum::warn!( target: LOG_TARGET, @@ -537,6 +548,16 @@ where c_messages, &metrics, ) + } else if expected_versions[PeerSet::Collation] == + Some(CollationVersion::VStaging.into()) + { + handle_peer_messages::( + remote, + PeerSet::Collation, + &mut shared.0.lock().collation_peers, + c_messages, + &metrics, + ) } else { gum::warn!( target: LOG_TARGET, @@ -827,15 +848,18 @@ fn update_our_view( ) }; - let filter_by_version = |peers: &[(PeerId, ProtocolVersion)], version| { - peers.iter().filter(|(_, v)| v == &version).map(|(p, _)| *p).collect::>() - }; + let v1_validation_peers = + filter_by_peer_version(&validation_peers, ValidationVersion::V1.into()); + let v1_collation_peers = filter_by_peer_version(&collation_peers, CollationVersion::V1.into()); - let v1_validation_peers = filter_by_version(&validation_peers, ValidationVersion::V1.into()); - let v1_collation_peers = filter_by_version(&collation_peers, CollationVersion::V1.into()); + let v2_validation_peers = + filter_by_peer_version(&validation_peers, ValidationVersion::V2.into()); + let v2_collation_peers = filter_by_peer_version(&collation_peers, CollationVersion::V2.into()); - let v2_validation_peers = filter_by_version(&validation_peers, ValidationVersion::V2.into()); - let v2_collation_peers = filter_by_version(&collation_peers, ValidationVersion::V2.into()); + let vstaging_validation_peers = + filter_by_peer_version(&validation_peers, ValidationVersion::VStaging.into()); + let vstaging_collation_peers = + filter_by_peer_version(&collation_peers, CollationVersion::VStaging.into()); send_validation_message_v1( net, @@ -853,7 +877,7 @@ fn update_our_view( metrics, ); - send_validation_message_vstaging( + send_validation_message_v2( net, v2_validation_peers, peerset_protocol_names, @@ -865,10 +889,25 @@ fn update_our_view( net, v2_collation_peers, peerset_protocol_names, - WireMessage::ViewUpdate(new_view), + WireMessage::ViewUpdate(new_view.clone()), + metrics, + ); + + send_validation_message_vstaging( + net, + vstaging_validation_peers, + peerset_protocol_names, + WireMessage::ViewUpdate(new_view.clone()), metrics, ); + send_collation_message_vstaging( + net, + vstaging_collation_peers, + peerset_protocol_names, + WireMessage::ViewUpdate(new_view.clone()), + metrics, + ); let our_view = OurView::new( live_heads.iter().take(MAX_VIEW_HEADS).cloned().map(|a| (a.hash, a.span)), finalized_number, diff --git a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs index ba9f5c0120e7..1c40759dde7f 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs @@ -596,7 +596,7 @@ fn declare_message( ); Versioned::V1(protocol_v1::CollationProtocol::CollatorProtocol(wire_message)) }, - CollationVersion::V2 | CollationVersion::VStaging => { + CollationVersion::V2 => { let declare_signature_payload = protocol_v2::declare_signature_payload(&state.local_peer_id); let wire_message = protocol_v2::CollatorProtocolMessage::Declare( @@ -606,6 +606,16 @@ fn declare_message( ); Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) }, + CollationVersion::VStaging => { + let declare_signature_payload = + protocol_vstaging::declare_signature_payload(&state.local_peer_id); + let wire_message = protocol_vstaging::CollatorProtocolMessage::Declare( + state.collator_pair.public(), + para_id, + state.collator_pair.sign(&declare_signature_payload), + ); + Versioned::VStaging(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) + }, }) } @@ -710,7 +720,7 @@ async fn advertise_collation( collation.status.advance_to_advertised(); let collation_message = match protocol_version { - CollationVersion::V2 | CollationVersion::VStaging => { + CollationVersion::V2 => { let wire_message = protocol_v2::CollatorProtocolMessage::AdvertiseCollation { relay_parent, candidate_hash: *candidate_hash, @@ -718,6 +728,16 @@ async fn advertise_collation( }; Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) }, + CollationVersion::VStaging => { + let wire_message = protocol_vstaging::CollatorProtocolMessage::AdvertiseCollation { + relay_parent, + candidate_hash: *candidate_hash, + parent_head_data_hash: collation.parent_head_data_hash, + }; + Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( + wire_message, + )) + }, CollationVersion::V1 => { let wire_message = protocol_v1::CollatorProtocolMessage::AdvertiseCollation(relay_parent); diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index bbc37d3ee2ae..0567b33dfe85 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -697,7 +697,8 @@ async fn request_collation( let requests = Requests::CollationFetchingV1(req); (requests, response_recv.boxed()) }, - (CollationVersion::V2, Some(ProspectiveCandidate { candidate_hash, .. })) => { + (CollationVersion::V2, Some(ProspectiveCandidate { candidate_hash, .. })) | + (CollationVersion::VStaging, Some(ProspectiveCandidate { candidate_hash, .. })) => { let (req, response_recv) = OutgoingRequest::new( Recipient::Peer(peer_id), request_v2::CollationFetchingRequest { relay_parent, para_id, candidate_hash }, diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index 329c380ed6a4..c628b1e6ce01 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -289,6 +289,12 @@ impl From for VersionedValidationProtocol { } } +impl From for VersionedValidationProtocol { + fn from(vstaging: vstaging::ValidationProtocol) -> Self { + VersionedValidationProtocol::VStaging(vstaging) + } +} + /// All supported versions of the collation protocol message. pub type VersionedCollationProtocol = Versioned; @@ -350,6 +356,7 @@ macro_rules! impl_versioned_try_from { match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out.clone())), Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out.clone())), + Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out.clone())), _ => Err(crate::WrongVariant), } } diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index a86f8ce9a4a7..0f1d9de2a565 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -154,6 +154,8 @@ impl PeerSet { Some("validation/1") } else if version == ValidationVersion::V2.into() { Some("validation/2") + } else if version == ValidationVersion::VStaging.into() { + Some("validation/3") } else { None }, @@ -162,6 +164,8 @@ impl PeerSet { Some("collation/1") } else if version == CollationVersion::V2.into() { Some("collation/2") + } else if version == CollationVersion::VStaging.into() { + Some("collation/3") } else { None }, @@ -238,7 +242,7 @@ pub enum CollationVersion { /// The second version. V2 = 2, /// Same format as V2, - VStaging, + VStaging = 3, } /// Marker indicating the version is unknown. diff --git a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs index f7f0834aba09..d9866af1ee23 100644 --- a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs +++ b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use net_protocol::{filter_by_peer_version, peer_set::ProtocolVersion}; use parity_scale_codec::Encode; use polkadot_node_network_protocol::{ @@ -1062,7 +1063,7 @@ async fn circulate_statement<'a, Context>( "We filter out duplicates above. qed.", ); - let (v1_peers_to_send, v2_peers_to_send) = peers_to_send + let (v1_peers_to_send, non_v1_peers_to_send) = peers_to_send .into_iter() .map(|peer_id| { let peer_data = @@ -1094,6 +1095,22 @@ async fn circulate_statement<'a, Context>( )) .await; } + + let peers_to_send: Vec<(PeerId, ProtocolVersion)> = non_v1_peers_to_send + .iter() + .map(|(p, _, version)| (*p, (*version).into())) + .collect(); + + let peer_needs_dependent_statement = v1_peers_to_send + .into_iter() + .chain(non_v1_peers_to_send) + .filter_map(|(peer, needs_dependent, _)| if needs_dependent { Some(peer) } else { None }) + .collect(); + + let v2_peers_to_send = filter_by_peer_version(&peers_to_send, ValidationVersion::V2.into()); + let vstaging_to_send = + filter_by_peer_version(&peers_to_send, ValidationVersion::VStaging.into()); + if !v2_peers_to_send.is_empty() { gum::trace!( target: LOG_TARGET, @@ -1103,17 +1120,28 @@ async fn circulate_statement<'a, Context>( "Sending statement to v2 peers", ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - v2_peers_to_send.iter().map(|(p, _, _)| *p).collect(), + v2_peers_to_send, compatible_v1_message(ValidationVersion::V2, payload.clone()).into(), )) .await; } - v1_peers_to_send - .into_iter() - .chain(v2_peers_to_send) - .filter_map(|(peer, needs_dependent, _)| if needs_dependent { Some(peer) } else { None }) - .collect() + if !vstaging_to_send.is_empty() { + gum::trace!( + target: LOG_TARGET, + ?vstaging_to_send, + ?relay_parent, + statement = ?stored.statement, + "Sending statement to vstaging peers", + ); + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + vstaging_to_send, + compatible_v1_message(ValidationVersion::VStaging, payload.clone()).into(), + )) + .await; + } + + peer_needs_dependent_statement } /// Send all statements about a given candidate hash to a peer. @@ -2171,7 +2199,10 @@ fn compatible_v1_message( ) -> net_protocol::StatementDistributionMessage { match version { ValidationVersion::V1 => Versioned::V1(message), - ValidationVersion::V2 | ValidationVersion::VStaging => + ValidationVersion::V2 => Versioned::V2(protocol_v2::StatementDistributionMessage::V1Compatibility(message)), + ValidationVersion::VStaging => Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::V1Compatibility(message), + ), } } diff --git a/polkadot/node/network/statement-distribution/src/v2/mod.rs b/polkadot/node/network/statement-distribution/src/v2/mod.rs index 51ae89eed781..0f29e33676b5 100644 --- a/polkadot/node/network/statement-distribution/src/v2/mod.rs +++ b/polkadot/node/network/statement-distribution/src/v2/mod.rs @@ -17,6 +17,7 @@ //! Implementation of the v2 statement distribution protocol, //! designed for asynchronous backing. +use net_protocol::{filter_by_peer_version, peer_set::ProtocolVersion}; use polkadot_node_network_protocol::{ self as net_protocol, grid_topology::SessionGridTopology, @@ -252,6 +253,7 @@ fn connected_validator_peer( struct PeerState { view: View, + protocol_version: ValidationVersion, implicit_view: HashSet, discovery_ids: Option>, } @@ -324,9 +326,13 @@ pub(crate) async fn handle_network_update( NetworkBridgeEvent::PeerConnected(peer_id, role, protocol_version, mut authority_ids) => { gum::trace!(target: LOG_TARGET, ?peer_id, ?role, ?protocol_version, "Peer connected"); - if protocol_version != ValidationVersion::V2.into() { + let versioned_protocol = if protocol_version != ValidationVersion::V2.into() && + protocol_version != ValidationVersion::VStaging.into() + { return - } + } else { + protocol_version.try_into().expect("Qed, we checked above") + }; if let Some(ref mut authority_ids) = authority_ids { authority_ids.retain(|a| match state.authorities.entry(a.clone()) { @@ -353,6 +359,7 @@ pub(crate) async fn handle_network_update( PeerState { view: View::default(), implicit_view: HashSet::new(), + protocol_version: versioned_protocol, discovery_ids: authority_ids, }, ); @@ -708,7 +715,7 @@ async fn send_peer_messages_for_relay_parent( send_pending_cluster_statements( ctx, relay_parent, - &peer, + &(peer, peer_data.protocol_version), validator_id, &mut local_validator_state.cluster_tracker, &state.candidates, @@ -720,7 +727,7 @@ async fn send_peer_messages_for_relay_parent( send_pending_grid_messages( ctx, relay_parent, - &peer, + &(peer, peer_data.protocol_version), validator_id, &per_session_state.groups, relay_parent_state, @@ -733,15 +740,26 @@ async fn send_peer_messages_for_relay_parent( fn pending_statement_network_message( statement_store: &StatementStore, relay_parent: Hash, - peer: &PeerId, + peer: &(PeerId, ValidationVersion), originator: ValidatorIndex, compact: CompactStatement, ) -> Option<(Vec, net_protocol::VersionedValidationProtocol)> { - statement_store - .validator_statement(originator, compact) - .map(|s| s.as_unchecked().clone()) - .map(|signed| protocol_v2::StatementDistributionMessage::Statement(relay_parent, signed)) - .map(|msg| (vec![*peer], Versioned::V2(msg).into())) + match peer.1 { + ValidationVersion::V2 => statement_store + .validator_statement(originator, compact) + .map(|s| s.as_unchecked().clone()) + .map(|signed| { + protocol_v2::StatementDistributionMessage::Statement(relay_parent, signed) + }) + .map(|msg| (vec![peer.0], Versioned::V2(msg).into())), + _ => statement_store + .validator_statement(originator, compact) + .map(|s| s.as_unchecked().clone()) + .map(|signed| { + protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, signed) + }) + .map(|msg| (vec![peer.0], Versioned::VStaging(msg).into())), + } } /// Send a peer all pending cluster statements for a relay parent. @@ -749,7 +767,7 @@ fn pending_statement_network_message( async fn send_pending_cluster_statements( ctx: &mut Context, relay_parent: Hash, - peer_id: &PeerId, + peer_id: &(PeerId, ValidationVersion), peer_validator_id: ValidatorIndex, cluster_tracker: &mut ClusterTracker, candidates: &Candidates, @@ -793,7 +811,7 @@ async fn send_pending_cluster_statements( async fn send_pending_grid_messages( ctx: &mut Context, relay_parent: Hash, - peer_id: &PeerId, + peer_id: &(PeerId, ValidationVersion), peer_validator_id: ValidatorIndex, groups: &Groups, relay_parent_state: &mut PerRelayParentState, @@ -855,20 +873,30 @@ async fn send_pending_grid_messages( candidate_hash, local_knowledge.clone(), ); - - messages.push(( - vec![*peer_id], - Versioned::V2( - protocol_v2::StatementDistributionMessage::BackedCandidateManifest( - manifest, - ), - ) - .into(), - )); + match peer_id.1 { + ValidationVersion::V2 => messages.push(( + vec![peer_id.0], + Versioned::V2( + protocol_v2::StatementDistributionMessage::BackedCandidateManifest( + manifest, + ), + ) + .into(), + )), + _ => messages.push(( + vec![peer_id.0], + Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest( + manifest, + ), + ) + .into(), + )), + }; }, grid::ManifestKind::Acknowledgement => { messages.extend(acknowledgement_and_statement_messages( - *peer_id, + peer_id, peer_validator_id, groups, relay_parent_state, @@ -1155,11 +1183,18 @@ async fn circulate_statement( (local_validator, targets) }; - let mut statement_to = Vec::new(); + let mut statement_to_peers: Vec<(PeerId, ProtocolVersion)> = Vec::new(); for (target, authority_id, kind) in targets { // Find peer ID based on authority ID, and also filter to connected. - let peer_id: PeerId = match authorities.get(&authority_id) { - Some(p) if peers.get(p).map_or(false, |p| p.knows_relay_parent(&relay_parent)) => *p, + let peer_id: (PeerId, ProtocolVersion) = match authorities.get(&authority_id) { + Some(p) if peers.get(p).map_or(false, |p| p.knows_relay_parent(&relay_parent)) => ( + *p, + peers + .get(p) + .expect("Qed, can't fail because it was checked above") + .protocol_version + .into(), + ), None | Some(_) => continue, }; @@ -1177,11 +1212,11 @@ async fn circulate_statement( originator, compact_statement.clone(), ); - statement_to.push(peer_id); + statement_to_peers.push(peer_id); } }, DirectTargetKind::Grid => { - statement_to.push(peer_id); + statement_to_peers.push(peer_id); local_validator.grid_tracker.sent_or_received_direct_statement( &per_session.groups, originator, @@ -1192,17 +1227,23 @@ async fn circulate_statement( } } + let statement_to_v2_peers = + filter_by_peer_version(&statement_to_peers, ValidationVersion::V2.into()); + + let statement_to_vstaging_peers = + filter_by_peer_version(&statement_to_peers, ValidationVersion::VStaging.into()); + // ship off the network messages to the network bridge. - if !statement_to.is_empty() { + if !statement_to_v2_peers.is_empty() { gum::debug!( target: LOG_TARGET, ?compact_statement, - n_peers = ?statement_to.len(), - "Sending statement to peers", + n_peers = ?statement_to_v2_peers.len(), + "Sending statement to v2 peers", ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - statement_to, + statement_to_v2_peers, Versioned::V2(protocol_v2::StatementDistributionMessage::Statement( relay_parent, statement.as_unchecked().clone(), @@ -1211,6 +1252,25 @@ async fn circulate_statement( )) .await; } + + if !statement_to_vstaging_peers.is_empty() { + gum::debug!( + target: LOG_TARGET, + ?compact_statement, + n_peers = ?statement_to_peers.len(), + "Sending statement to vstaging peers", + ); + + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + statement_to_vstaging_peers, + Versioned::VStaging(protocol_vstaging::StatementDistributionMessage::Statement( + relay_parent, + statement.as_unchecked().clone(), + )) + .into(), + )) + .await; + } } /// Check a statement signature under this parent hash. fn check_statement_signature( @@ -1696,14 +1756,8 @@ async fn provide_candidate_to_grid( statement_knowledge: filter.clone(), }; - let manifest_message = - Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateManifest(manifest)); - let ack_message = Versioned::V2( - protocol_v2::StatementDistributionMessage::BackedCandidateKnown(acknowledgement), - ); - - let mut manifest_peers = Vec::new(); - let mut ack_peers = Vec::new(); + let mut manifest_peers: Vec<(PeerId, ProtocolVersion)> = Vec::new(); + let mut ack_peers: Vec<(PeerId, ProtocolVersion)> = Vec::new(); let mut post_statements = Vec::new(); for (v, action) in actions { @@ -1711,7 +1765,7 @@ async fn provide_candidate_to_grid( None => continue, Some(p) => if peers.get(&p).map_or(false, |d| d.knows_relay_parent(&relay_parent)) { - p + (p, peers.get(&p).expect("Qed, was checked above").protocol_version.into()) } else { continue }, @@ -1737,42 +1791,91 @@ async fn provide_candidate_to_grid( &per_session.groups, group_index, candidate_hash, + &(p.0, p.1.try_into().expect("Qed, can not fail was checked above")), ) .into_iter() - .map(|m| (vec![p], m)), + .map(|m| (vec![p.0], m)), ); } - if !manifest_peers.is_empty() { + let manifest_peers_v2 = filter_by_peer_version(&manifest_peers, ValidationVersion::V2.into()); + let manifest_peers_vstaging = + filter_by_peer_version(&manifest_peers, ValidationVersion::VStaging.into()); + if !manifest_peers_v2.is_empty() { gum::debug!( target: LOG_TARGET, ?candidate_hash, - n_peers = manifest_peers.len(), - "Sending manifest to peers" + n_peers = manifest_peers_v2.len(), + "Sending manifest to v2 peers" ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - manifest_peers, - manifest_message.into(), + manifest_peers_v2, + Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateManifest( + manifest.clone(), + )) + .into(), )) .await; } - if !ack_peers.is_empty() { + if !manifest_peers_vstaging.is_empty() { gum::debug!( target: LOG_TARGET, ?candidate_hash, - n_peers = ack_peers.len(), - "Sending acknowledgement to peers" + n_peers = manifest_peers_vstaging.len(), + "Sending manifest to vstaging peers" ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - ack_peers, - ack_message.into(), + manifest_peers_vstaging, + Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(manifest), + ) + .into(), )) .await; } + let ack_peers_v2 = filter_by_peer_version(&ack_peers, ValidationVersion::V2.into()); + let ack_peers_vstaging = filter_by_peer_version(&ack_peers, ValidationVersion::VStaging.into()); + if !ack_peers_v2.is_empty() { + gum::debug!( + target: LOG_TARGET, + ?candidate_hash, + n_peers = ack_peers_v2.len(), + "Sending acknowledgement to v2 peers" + ); + + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + ack_peers_v2, + Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement.clone(), + )) + .into(), + )) + .await; + } + + if !ack_peers_vstaging.is_empty() { + gum::debug!( + target: LOG_TARGET, + ?candidate_hash, + n_peers = ack_peers_vstaging.len(), + "Sending acknowledgement to vstaging peers" + ); + + ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( + ack_peers_vstaging, + Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement, + ), + ) + .into(), + )) + .await; + } if !post_statements.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessages(post_statements)) .await; @@ -2055,6 +2158,7 @@ fn post_acknowledgement_statement_messages( groups: &Groups, group_index: GroupIndex, candidate_hash: CandidateHash, + peer: &(PeerId, ValidationVersion), ) -> Vec { let sending_filter = match grid_tracker.pending_statements_for(recipient, candidate_hash) { None => return Vec::new(), @@ -2071,14 +2175,22 @@ fn post_acknowledgement_statement_messages( recipient, statement.payload(), ); - - messages.push(Versioned::V2( - protocol_v2::StatementDistributionMessage::Statement( - relay_parent, - statement.as_unchecked().clone(), - ) - .into(), - )); + match peer.1.into() { + ValidationVersion::V2 => messages.push(Versioned::V2( + protocol_v2::StatementDistributionMessage::Statement( + relay_parent, + statement.as_unchecked().clone(), + ) + .into(), + )), + _ => messages.push(Versioned::VStaging( + protocol_vstaging::StatementDistributionMessage::Statement( + relay_parent, + statement.as_unchecked().clone(), + ) + .into(), + )), + }; } messages @@ -2148,7 +2260,15 @@ async fn handle_incoming_manifest( }; let messages = acknowledgement_and_statement_messages( - peer, + &( + peer, + state + .peers + .get(&peer) + .map(|val| val.protocol_version) + // Assume the latest stable version, if we don't have info about peer version. + .unwrap_or(ValidationVersion::V2), + ), sender_index, &per_session.groups, relay_parent_state, @@ -2179,7 +2299,7 @@ async fn handle_incoming_manifest( /// Produces acknowledgement and statement messages to be sent over the network, /// noting that they have been sent within the grid topology tracker as well. fn acknowledgement_and_statement_messages( - peer: PeerId, + peer: &(PeerId, ValidationVersion), validator_index: ValidatorIndex, groups: &Groups, relay_parent_state: &mut PerRelayParentState, @@ -2198,11 +2318,20 @@ fn acknowledgement_and_statement_messages( statement_knowledge: local_knowledge.clone(), }; - let msg = Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( - acknowledgement, + let msg_v2 = Versioned::V2(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement.clone(), )); - let mut messages = vec![(vec![peer], msg.into())]; + let mut messages = match peer.1 { + ValidationVersion::V2 => vec![(vec![peer.0], msg_v2.into())], + _ => vec![( + vec![peer.0], + Versioned::VStaging(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement, + )) + .into(), + )], + }; local_validator.grid_tracker.manifest_sent_to( groups, @@ -2219,9 +2348,10 @@ fn acknowledgement_and_statement_messages( &groups, group_index, candidate_hash, + peer, ); - messages.extend(statement_messages.into_iter().map(|m| (vec![peer], m))); + messages.extend(statement_messages.into_iter().map(|m| (vec![peer.0], m))); messages } @@ -2301,6 +2431,15 @@ async fn handle_incoming_acknowledgement( &per_session.groups, group_index, candidate_hash, + &( + peer, + state + .peers + .get(&peer) + .map(|val| val.protocol_version) + // Assume the latest stable version, if we don't have info about peer version. + .unwrap_or(ValidationVersion::V2), + ), ); if !messages.is_empty() { diff --git a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml index 3fbed4046bde..5ee088dcc082 100644 --- a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml @@ -39,3 +39,6 @@ sc-service = { path = "../../../../../substrate/client/service" } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } tokio = { version = "1.24.2", features = ["macros"] } + +[features] +network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] \ No newline at end of file diff --git a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml index ab7fa0195d13..0b75b1a48743 100644 --- a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml +++ b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml @@ -2,7 +2,7 @@ timeout = 1000 bootnode = true -[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] +[relaychain.genesis.runtime.configuration.config] max_validators_per_core = 1 needed_approvals = 7 relay_vrf_modulo_samples = 5 From 55797197862d968f831d8f7a3ec91c53eb3a4811 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 11 Oct 2023 10:58:07 +0200 Subject: [PATCH 32/89] Fix cargo fmt Signed-off-by: Alexandru Gheorghe --- polkadot/cli/Cargo.toml | 2 +- polkadot/node/network/protocol/Cargo.toml | 2 +- polkadot/node/network/protocol/src/lib.rs | 3 ++- polkadot/node/service/Cargo.toml | 2 +- polkadot/parachain/test-parachains/undying/collator/Cargo.toml | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/polkadot/cli/Cargo.toml b/polkadot/cli/Cargo.toml index ca1c954ba47c..8abc09dd4e26 100644 --- a/polkadot/cli/Cargo.toml +++ b/polkadot/cli/Cargo.toml @@ -74,4 +74,4 @@ runtime-metrics = [ "service/runtime-metrics", ] -network-protocol-staging = [ "service/network-protocol-staging" ] \ No newline at end of file +network-protocol-staging = [ "service/network-protocol-staging" ] diff --git a/polkadot/node/network/protocol/Cargo.toml b/polkadot/node/network/protocol/Cargo.toml index f4a64d274060..c33b9eae3252 100644 --- a/polkadot/node/network/protocol/Cargo.toml +++ b/polkadot/node/network/protocol/Cargo.toml @@ -29,4 +29,4 @@ bitvec = "1" rand_chacha = "0.3.1" [features] -network-protocol-staging = [] \ No newline at end of file +network-protocol-staging = [] diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index c628b1e6ce01..f01de9862ce8 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -356,7 +356,8 @@ macro_rules! impl_versioned_try_from { match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out.clone())), Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out.clone())), - Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out.clone())), + Versioned::VStaging($vstaging_pat) => + Ok(Versioned::VStaging($vstaging_out.clone())), _ => Err(crate::WrongVariant), } } diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index d3a7b2f15725..ee092e277332 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -226,4 +226,4 @@ runtime-metrics = [ network-protocol-staging = [ "polkadot-node-network-protocol/network-protocol-staging", -] \ No newline at end of file +] diff --git a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml index b694895f3743..578c3d6715dc 100644 --- a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml @@ -41,4 +41,4 @@ sp-keyring = { path = "../../../../../substrate/primitives/keyring" } tokio = { version = "1.24.2", features = ["macros"] } [features] -network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] \ No newline at end of file +network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] From ee3b70267d53d25e330f355b8523f98410437b6e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Oct 2023 15:47:25 +0300 Subject: [PATCH 33/89] approval-distribution: fix unittests Signed-off-by: Alexandru Gheorghe --- .../approval-distribution/src/tests.rs | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index 58601de327cd..27514277986d 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -873,8 +873,9 @@ fn import_approval_happy_path_v1_v2_peers() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_b, msg).await; + let msg: protocol_vstaging::ApprovalDistributionMessage = + protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_vstaging(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -975,7 +976,7 @@ fn import_approval_happy_path_v2() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_b, msg).await; + send_message_from_peer_vstaging(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1059,7 +1060,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { assignment, (0 as CandidateIndex).into(), )]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1101,7 +1102,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { (1 as CandidateIndex).into(), )]); - send_message_from_peer_v2(overseer, &peer_c, msg).await; + send_message_from_peer_vstaging(overseer, &peer_c, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1137,7 +1138,7 @@ fn multiple_assignments_covered_with_one_approval_vote() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1241,7 +1242,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { assignment, (0 as CandidateIndex).into(), )]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1268,7 +1269,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { (1 as CandidateIndex).into(), )]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1289,7 +1290,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_d, msg).await; + send_message_from_peer_vstaging(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1410,7 +1411,7 @@ fn import_approval_bad() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_b, msg).await; + send_message_from_peer_vstaging(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; @@ -1436,7 +1437,7 @@ fn import_approval_bad() { // and try again let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, &peer_b, msg).await; + send_message_from_peer_vstaging(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1762,7 +1763,7 @@ fn import_remotely_then_locally() { signature: dummy_signature(), }; let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_v2(overseer, peer, msg).await; + send_message_from_peer_vstaging(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -3172,12 +3173,13 @@ fn resends_messages_periodically() { /// Tests that peers correctly receive versioned messages. #[test] fn import_versioned_approval() { - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - let peer_c = PeerId::random(); + let peers = make_peers_and_authority_ids(15); + let peer_a = peers.get(0).unwrap().0; + let peer_b = peers.get(1).unwrap().0; + let peer_c = peers.get(2).unwrap().0; + let parent_hash = Hash::repeat_byte(0xFF); let hash = Hash::repeat_byte(0xAA); - let state = state_without_reputation_delay(); let _ = test_harness(state, |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; @@ -3186,6 +3188,10 @@ fn import_versioned_approval() { setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V1).await; setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V2).await; + // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under + // testing. + setup_gossip_topology(overseer, make_gossip_topology(1, &peers, &[0, 1], &[2, 4], 3)).await; + // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { hash, @@ -3253,7 +3259,7 @@ fn import_versioned_approval() { vote, tx, )) => { - assert_eq!(vote, approval); + assert_eq!(vote, approval.into()); tx.send(ApprovalCheckResult::Accepted).unwrap(); } ); @@ -3273,6 +3279,7 @@ fn import_versioned_approval() { assert_eq!(approvals.len(), 1); } ); + assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( From 0d2fbba53313a9c1e58e99e224e59624808dd7b4 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Oct 2023 17:44:30 +0300 Subject: [PATCH 34/89] Approval voting fixup Signed-off-by: Alexandru Gheorghe --- .../functional/0006-approval-voting-coalescing.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml index ec6d17dff4e7..acc9290eb7c8 100644 --- a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml @@ -6,11 +6,11 @@ default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" -[relaychain.genesis.runtime.runtime_genesis_config.configuration.config] +[relaychain.genesis.runtime.configuration.config] needed_approvals = 5 relay_vrf_modulo_samples = 3 -[relaychain.genesis.runtime.runtime_genesis_config.configuration.config.approval_voting_params] +[relaychain.genesis.runtime.configuration.config.approval_voting_params] max_approval_coalesce_count = 6 [relaychain.default_resources] From e672c1daeb50428a537ee8c6a790407b4bd8fff2 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 16 Oct 2023 19:12:17 +0300 Subject: [PATCH 35/89] Fixup runtime api post merge Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/runtime-api/src/lib.rs | 2 +- .../subsystem-types/src/runtime_client.rs | 1 + polkadot/primitives/src/runtime_api.rs | 2 +- polkadot/runtime/rococo/src/lib.rs | 21 ++++++++++++------ polkadot/runtime/westend/src/lib.rs | 22 ++++++++++++------- .../0006-approval-voting-coalescing.toml | 2 +- 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/polkadot/node/core/runtime-api/src/lib.rs b/polkadot/node/core/runtime-api/src/lib.rs index 9610d6b57fb0..1cd6d4ced267 100644 --- a/polkadot/node/core/runtime-api/src/lib.rs +++ b/polkadot/node/core/runtime-api/src/lib.rs @@ -558,7 +558,7 @@ where sender ), Request::ApprovalVotingParams(sender) => { - query!(ApprovalVotingParams, approval_voting_params(), ver = 6, sender) + query!(ApprovalVotingParams, approval_voting_params(), ver = 8, sender) }, Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) => query!( SubmitReportDisputeLost, diff --git a/polkadot/node/subsystem-types/src/runtime_client.rs b/polkadot/node/subsystem-types/src/runtime_client.rs index 18ef31c02e14..141b2108eb9a 100644 --- a/polkadot/node/subsystem-types/src/runtime_client.rs +++ b/polkadot/node/subsystem-types/src/runtime_client.rs @@ -256,6 +256,7 @@ pub trait RuntimeApiSubsystemClient { para_id: Id, ) -> Result, ApiError>; + // == v8: Approval voting params == /// Approval voting configuration parameters async fn approval_voting_params(&self, at: Hash) -> Result; } diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index c178730511c8..b242e04d3425 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -259,7 +259,7 @@ sp_api::decl_runtime_apis! { fn async_backing_params() -> AsyncBackingParams; /// Approval voting configuration parameters - #[api_version(99)] + #[api_version(8)] fn approval_voting_params() -> ApprovalVotingParams; } } diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 0a36a41517b7..2285a7928521 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -23,11 +23,12 @@ use pallet_nis::WithMaximumOf; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, claims, crowdloan, impl_runtime_weights, @@ -49,7 +50,9 @@ use runtime_parachains::{ inclusion::{AggregateMessageOrigin, UmpQueueId}, initializer as parachains_initializer, origin as parachains_origin, paras as parachains_paras, paras_inherent as parachains_paras_inherent, - runtime_api_impl::v7 as parachains_runtime_api_impl, + runtime_api_impl::{ + v7 as parachains_runtime_api_impl, vstaging as parachains_runtime_api_impl_staging, + }, scheduler as parachains_scheduler, session_info as parachains_session_info, shared as parachains_shared, }; @@ -1586,7 +1589,7 @@ sp_api::impl_runtime_apis! { } } - #[api_version(7)] + #[api_version(8)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() @@ -1729,6 +1732,10 @@ sp_api::impl_runtime_apis! { fn async_backing_params() -> primitives::AsyncBackingParams { parachains_runtime_api_impl::async_backing_params::() } + + fn approval_voting_params() -> ApprovalVotingParams { + parachains_runtime_api_impl_staging::approval_voting_params::() + } } #[api_version(3)] diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 37accc166262..26ee750f0ec3 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -45,12 +45,12 @@ use pallet_session::historical as session_historical; use pallet_transaction_payment::{CurrencyAdapter, FeeDetails, RuntimeDispatchInfo}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, PARACHAIN_KEY_TYPE_ID, + slashing, vstaging::ApprovalVotingParams, AccountId, AccountIndex, Balance, BlockNumber, + CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, + ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, + PvfCheckStatement, ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, + ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, PARACHAIN_KEY_TYPE_ID, }; use runtime_common::{ assigned_slots, auctions, crowdloan, @@ -70,7 +70,9 @@ use runtime_parachains::{ inclusion::{AggregateMessageOrigin, UmpQueueId}, initializer as parachains_initializer, origin as parachains_origin, paras as parachains_paras, paras_inherent as parachains_paras_inherent, reward_points as parachains_reward_points, - runtime_api_impl::v7 as parachains_runtime_api_impl, + runtime_api_impl::{ + v7 as parachains_runtime_api_impl, vstaging as parachains_runtime_api_impl_staging, + }, scheduler as parachains_scheduler, session_info as parachains_session_info, shared as parachains_shared, }; @@ -1696,7 +1698,7 @@ sp_api::impl_runtime_apis! { } } - #[api_version(7)] + #[api_version(8)] impl primitives::runtime_api::ParachainHost for Runtime { fn validators() -> Vec { parachains_runtime_api_impl::validators::() @@ -1839,6 +1841,10 @@ sp_api::impl_runtime_apis! { fn async_backing_params() -> primitives::AsyncBackingParams { parachains_runtime_api_impl::async_backing_params::() } + + fn approval_voting_params() -> ApprovalVotingParams { + parachains_runtime_api_impl_staging::approval_voting_params::() + } } impl beefy_primitives::BeefyApi for Runtime { diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml index acc9290eb7c8..99aa723ec191 100644 --- a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml @@ -11,7 +11,7 @@ chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default relay_vrf_modulo_samples = 3 [relaychain.genesis.runtime.configuration.config.approval_voting_params] - max_approval_coalesce_count = 6 + max_approval_coalesce_count = 5 [relaychain.default_resources] limits = { memory = "4G", cpu = "2" } From deccd9861ee4c1c4ab92c00d899ebfafebeba274 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 12:23:03 +0300 Subject: [PATCH 36/89] Fixup clippy Signed-off-by: Alexandru Gheorghe --- polkadot/node/service/src/parachains_db/upgrade.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index b99f885176b0..a22ec675d0e6 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -595,7 +595,7 @@ mod tests { // We need to properly set db version for upgrade to work. fs::write(version_file_path(db_dir.path()), "3").expect("Failed to write DB version"); let expected_candidates = { - let db = Database::open(&db_cfg, db_path.clone()).unwrap(); + let db = Database::open(&db_cfg, db_path).unwrap(); assert_eq!(db.num_columns(), super::columns::v3::NUM_COLUMNS as u32); let db = DbAdapter::new(db, columns::v3::ORDERED_COL); // Fill the approval voting column with test data. From cc5a8fee46d89978af9a8cf7b29123ff5f3d6ca5 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 15:03:19 +0300 Subject: [PATCH 37/89] Minor cleanups Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 2 +- polkadot/node/core/pvf/src/host.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index cb46782d4878..0b95ebbad7b7 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -3541,7 +3541,7 @@ fn compute_delayed_approval_sending_tick( .approval_entry(&block_entry.block_hash()) .and_then(|approval_entry| approval_entry.our_assignment()) .map(|our_assignment| our_assignment.tranche()) - .unwrap(); + .unwrap_or_default(); let assignment_triggered_tick = current_block_tick + assignment_tranche as Tick; diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index fc46622dba91..81695829122b 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -187,8 +187,7 @@ impl Config { prepare_workers_hard_max_num: 1, execute_worker_program_path, execute_worker_spawn_timeout: Duration::from_secs(3), - // TODO: cleanup increased for versi experimenting. - execute_workers_max_num: 4, + execute_workers_max_num: 2, } } } From e24d8894d541b4f0c50ab45afa24f89333707b88 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 15:24:27 +0300 Subject: [PATCH 38/89] Add metric to see average delayed tick Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 0b95ebbad7b7..2a59796b5470 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -181,6 +181,7 @@ struct MetricsInner { approved_by_one_third: prometheus::Counter, wakeups_triggered_total: prometheus::Counter, coalesced_approvals_buckets: prometheus::Histogram, + coalesced_approvals_waiting_times: prometheus::Histogram, candidate_approval_time_ticks: prometheus::Histogram, block_approval_time_ticks: prometheus::Histogram, time_db_transaction: prometheus::Histogram, @@ -216,6 +217,12 @@ impl Metrics { } } + fn on_delayed_approval(&self, delayed_ticks: u64) { + if let Some(metrics) = &self.0 { + metrics.coalesced_approvals_waiting_times.observe(delayed_ticks as f64) + } + } + fn on_approval_stale(&self) { if let Some(metrics) = &self.0 { metrics.approvals_produced_total.with_label_values(&["stale"]).inc() @@ -373,6 +380,15 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + coalesced_approvals_waiting_times: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_approvals_coalesced_approvals_waiting_times", + "Number of ticks we delay the sending of a candidate approval", + ).buckets(vec![1.1, 2.1, 3.1, 4.1, 6.1, 8.1, 12.1, 20.1, 32.1]), + )?, + registry, + )?, approved_by_one_third: prometheus::register( prometheus::Counter::new( "polkadot_parachain_approved_by_one_third", @@ -3297,6 +3313,7 @@ async fn issue_approval( &block_entry, &candidate_entry, session_info, + &metrics, ), ) .is_some() @@ -3535,6 +3552,7 @@ fn compute_delayed_approval_sending_tick( block_entry: &BlockEntry, candidate_entry: &CandidateEntry, session_info: &SessionInfo, + metrics: &Metrics, ) -> Tick { let current_block_tick = slot_number_to_tick(state.slot_duration_millis, block_entry.slot()); let assignment_tranche = candidate_entry @@ -3551,11 +3569,14 @@ fn compute_delayed_approval_sending_tick( ); let tick_now = state.clock.tick_now(); - min( + let sign_no_later_than = min( tick_now + MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick, // We don't want to accidentally cause, no-shows so if we are past // the seconnd half of the no show time, force the sending of the // approval immediately. assignment_triggered_tick + no_show_duration_ticks / 2, - ) + ); + + metrics.on_delayed_approval(sign_no_later_than.checked_sub(tick_now).unwrap_or_default()); + sign_no_later_than } From eebde478bc5f8a87d419ab160698d63be0b72051 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 16:06:23 +0300 Subject: [PATCH 39/89] Fixup CI Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/runtime-api/src/tests.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/polkadot/node/core/runtime-api/src/tests.rs b/polkadot/node/core/runtime-api/src/tests.rs index d6875194bccf..5207d03a412d 100644 --- a/polkadot/node/core/runtime-api/src/tests.rs +++ b/polkadot/node/core/runtime-api/src/tests.rs @@ -20,13 +20,12 @@ use polkadot_node_primitives::{BabeAllowedSlots, BabeEpoch, BabeEpochConfigurati use polkadot_node_subsystem::SpawnGlue; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_primitives::{ - async_backing, slashing, - vstaging::{self, ApprovalVotingParams}, - AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, - PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, - Slot, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, + async_backing, slashing, vstaging::ApprovalVotingParams, AuthorityDiscoveryId, BlockNumber, + CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, + DisputeState, ExecutorParams, GroupRotationInfo, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, + ScrapedOnChainVotes, SessionIndex, SessionInfo, Slot, ValidationCode, ValidationCodeHash, + ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_api::ApiError; use sp_core::testing::TaskExecutor; From f400d5af1143abb920844e3286d8d64c654afeb2 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 17 Oct 2023 17:52:52 +0300 Subject: [PATCH 40/89] ApprovalVotingParams sane default Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 2a59796b5470..869ae9518166 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -892,7 +892,7 @@ impl State { target: LOG_TARGET, "Could not request approval voting params from runtime using defaults" ); - ApprovalVotingParams { max_approval_coalesce_count: 6 } + ApprovalVotingParams { max_approval_coalesce_count: 1 } }, } } From 0a952c0ed309e09d3651c44744ae82bde1ca8514 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 18 Oct 2023 15:41:11 +0300 Subject: [PATCH 41/89] Add zombienet to check approval coalescing Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/zombienet/polkadot.yml | 8 ++++++ ...l => 0007-approval-voting-coalescing.toml} | 4 +-- ... => 0007-approval-voting-coalescing.zndsl} | 27 ++++++++++++------- 3 files changed, 28 insertions(+), 11 deletions(-) rename polkadot/zombienet_tests/functional/{0006-approval-voting-coalescing.toml => 0007-approval-voting-coalescing.toml} (98%) rename polkadot/zombienet_tests/functional/{0006-approval-voting-coalescing.zndsl => 0007-approval-voting-coalescing.zndsl} (70%) diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 8fc8b280bba8..1a7b45e52d83 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -113,6 +113,14 @@ zombienet-polkadot-functional-0006-parachains-max-tranche0: --local-dir="${LOCAL_DIR}/functional" --test="0006-parachains-max-tranche0.zndsl" +zombienet-polkadot-functional-0007-approval-voting-coalescing: + extends: + - .zombienet-polkadot-common + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh + --local-dir="${LOCAL_DIR}/functional" + --test="0007-approval-voting-coalescing.zndsl" + zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: - .zombienet-polkadot-common diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml similarity index 98% rename from polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml rename to polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index 99aa723ec191..c7eeeb5383e0 100644 --- a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -8,7 +8,7 @@ chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default [relaychain.genesis.runtime.configuration.config] needed_approvals = 5 - relay_vrf_modulo_samples = 3 + relay_vrf_modulo_samples = 6 [relaychain.genesis.runtime.configuration.config.approval_voting_params] max_approval_coalesce_count = 5 @@ -19,7 +19,7 @@ requests = { memory = "2G", cpu = "1" } [[relaychain.nodes]] name = "alice" - args = [ "--alice", "-lparachain=debug,runtime=debug,peerset=trace" ] + args = [ "--alice", "-lparachain=trace,runtime=debug" ] [[relaychain.nodes]] name = "bob" diff --git a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl similarity index 70% rename from polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl rename to polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl index 272ac4fa2640..a4cc454f38cc 100644 --- a/polkadot/zombienet_tests/functional/0006-approval-voting-coalescing.zndsl +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl @@ -1,5 +1,5 @@ Description: Approval voting coalescing does not lag finality -Network: ./0006-approval-voting-coalescing.toml +Network: ./0007-approval-voting-coalescing.toml Creds: config # Check authority status. @@ -41,11 +41,20 @@ ferdie: reports substrate_block_height{status="finalized"} is at least 30 within one: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds two: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -alice: reports polkadot_parachain_approval_checking_finality_lag is 0 -bob: reports polkadot_parachain_approval_checking_finality_lag is 0 -charlie: reports polkadot_parachain_approval_checking_finality_lag is 0 -dave: reports polkadot_parachain_approval_checking_finality_lag is 0 -ferdie: reports polkadot_parachain_approval_checking_finality_lag is 0 -eve: reports polkadot_parachain_approval_checking_finality_lag is 0 -one: reports polkadot_parachain_approval_checking_finality_lag is 0 -two: reports polkadot_parachain_approval_checking_finality_lag is 0 \ No newline at end of file +alice: reports polkadot_parachain_approval_checking_finality_lag < 3 +bob: reports polkadot_parachain_approval_checking_finality_lag < 3 +charlie: reports polkadot_parachain_approval_checking_finality_lag < 3 +dave: reports polkadot_parachain_approval_checking_finality_lag < 3 +ferdie: reports polkadot_parachain_approval_checking_finality_lag < 3 +eve: reports polkadot_parachain_approval_checking_finality_lag < 3 +one: reports polkadot_parachain_approval_checking_finality_lag < 3 +two: reports polkadot_parachain_approval_checking_finality_lag < 3 + +alice: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +bob: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +charlie: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +dave: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +ferdie: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +eve: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +one: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +two: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds \ No newline at end of file From 183ba64aff9a48ef20701e7814c401156be77447 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 12:48:50 +0300 Subject: [PATCH 42/89] Fix review typo Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/backend.rs | 2 +- polkadot/node/core/approval-voting/src/lib.rs | 2 +- polkadot/node/core/approval-voting/src/persisted_entries.rs | 2 +- polkadot/node/network/approval-distribution/src/lib.rs | 2 +- polkadot/node/service/src/parachains_db/upgrade.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/backend.rs b/polkadot/node/core/approval-voting/src/backend.rs index 374e7a826d19..d98f3c5fd202 100644 --- a/polkadot/node/core/approval-voting/src/backend.rs +++ b/polkadot/node/core/approval-voting/src/backend.rs @@ -66,7 +66,7 @@ pub trait Backend { I: IntoIterator; } -/// A read only backed to enable db migration from version 1 of DB. +/// A read only backend to enable db migration from version 1 of DB. pub trait V1ReadBackend: Backend { /// Load a candidate entry from the DB with scheme version 1. fn load_candidate_entry_v1( diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 4ec006a1d81c..e7cbde567392 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -2146,7 +2146,7 @@ where // Since we don't account for tranche in distribution message fingerprinting, some // validators can be assigned to the same core (VRF modulo vs VRF delay). These can be - // safely ignored ignored. However, if an assignment is for multiple cores (these are only + // safely ignored. However, if an assignment is for multiple cores (these are only // tranche0), we cannot ignore it, because it would mean ignoring other non duplicate // assignments. if is_duplicate { diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 155b2f9c4e02..9cfe1c4cf8da 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -353,7 +353,7 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, - // A list of assignments for which wea already distributed the assignment. + // A list of assignments for which we already distributed the assignment. // We use this to ensure we don't distribute multiple core assignments twice as we track // individual wakeups for each core. distributed_assignments: Bitfield, diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 87e8a072bf68..46d687035f51 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -188,7 +188,7 @@ impl ApprovalEntry { self.routing_info.required_routing = required_routing; } - // Records a new approval. Returns false if the claimed candidate is not found or we already + // Records a new approval. Returns error if the claimed candidate is not found or we already // have received the approval. pub fn note_approval( &mut self, diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index a22ec675d0e6..78e53a251eed 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -63,7 +63,7 @@ pub(crate) fn try_upgrade_db( db_kind: DatabaseKind, target_version: Version, ) -> Result<(), Error> { - // Ensure we don't loop forever below befcause of a bug. + // Ensure we don't loop forever below because of a bug. const MAX_MIGRATIONS: u32 = 30; #[cfg(test)] From 3fe8c0d6561dbdd71b4e36f8b22b15d8e7db44c8 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 12:56:33 +0300 Subject: [PATCH 43/89] Remove file accidentally added which was removed from master Signed-off-by: Alexandru Gheorghe --- polkadot/runtime/polkadot/src/lib.rs | 2621 -------------------------- 1 file changed, 2621 deletions(-) delete mode 100644 polkadot/runtime/polkadot/src/lib.rs diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs deleted file mode 100644 index 9c84c8840cd5..000000000000 --- a/polkadot/runtime/polkadot/src/lib.rs +++ /dev/null @@ -1,2621 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! The Polkadot runtime. This can be compiled with `#[no_std]`, ready for Wasm. - -#![cfg_attr(not(feature = "std"), no_std)] -// `construct_runtime!` does a lot of recursion and requires us to increase the limit to 256. -#![recursion_limit = "512"] - -use pallet_transaction_payment::CurrencyAdapter; -use runtime_common::{ - auctions, claims, crowdloan, impl_runtime_weights, impls::DealWithFees, paras_registrar, - prod_or_fast, slots, BlockHashCount, BlockLength, CurrencyToVote, SlowAdjustingFeeUpdate, -}; - -use runtime_parachains::{ - assigner_parachains as parachains_assigner_parachains, - configuration as parachains_configuration, disputes as parachains_disputes, - disputes::slashing as parachains_slashing, - dmp as parachains_dmp, hrmp as parachains_hrmp, inclusion as parachains_inclusion, - inclusion::{AggregateMessageOrigin, UmpQueueId}, - initializer as parachains_initializer, origin as parachains_origin, paras as parachains_paras, - paras_inherent as parachains_paras_inherent, reward_points as parachains_reward_points, - runtime_api_impl::v5 as parachains_runtime_api_impl, - scheduler as parachains_scheduler, session_info as parachains_session_info, - shared as parachains_shared, -}; - -use authority_discovery_primitives::AuthorityId as AuthorityDiscoveryId; -use beefy_primitives::ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefySignature}; -use frame_election_provider_support::{ - bounds::ElectionBoundsBuilder, generate_solution_type, onchain, SequentialPhragmen, -}; -use frame_support::{ - construct_runtime, parameter_types, - traits::{ - fungible::HoldConsideration, ConstU32, Contains, EitherOf, EitherOfDiverse, EverythingBut, - InstanceFilter, KeyOwnerProofSystem, LinearStoragePrice, PrivilegeCmp, ProcessMessage, - ProcessMessageError, WithdrawReasons, - }, - weights::{ConstantMultiplier, WeightMeter}, - PalletId, -}; -use frame_system::EnsureRoot; -use pallet_grandpa::{fg_primitives, AuthorityId as GrandpaId}; -use pallet_im_online::sr25519::AuthorityId as ImOnlineId; -use pallet_session::historical as session_historical; -use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; -use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; -use primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, - Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, Nonce, - OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, SessionInfo, Signature, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, LOWEST_PUBLIC_ID, - PARACHAIN_KEY_TYPE_ID, -}; -use sp_core::OpaqueMetadata; -use sp_mmr_primitives as mmr; -use sp_runtime::{ - create_runtime_str, - curve::PiecewiseLinear, - generic, impl_opaque_keys, - traits::{ - AccountIdLookup, BlakeTwo256, Block as BlockT, ConvertInto, Extrinsic as ExtrinsicT, - OpaqueKeys, SaturatedConversion, Verify, - }, - transaction_validity::{TransactionPriority, TransactionSource, TransactionValidity}, - ApplyExtrinsicResult, FixedU128, KeyTypeId, Perbill, Percent, Permill, RuntimeDebug, -}; -use sp_staking::SessionIndex; -use sp_std::{cmp::Ordering, collections::btree_map::BTreeMap, prelude::*}; -#[cfg(any(feature = "std", test))] -use sp_version::NativeVersion; -use sp_version::RuntimeVersion; -use xcm::latest::Junction; - -pub use frame_system::Call as SystemCall; -pub use pallet_balances::Call as BalancesCall; -pub use pallet_election_provider_multi_phase::{Call as EPMCall, GeometricDepositBase}; -#[cfg(feature = "std")] -pub use pallet_staking::StakerStatus; -use pallet_staking::UseValidatorsMap; -pub use pallet_timestamp::Call as TimestampCall; -use sp_runtime::traits::Get; -#[cfg(any(feature = "std", test))] -pub use sp_runtime::BuildStorage; - -/// Constant values used within the runtime. -use polkadot_runtime_constants::{currency::*, fee::*, time::*}; - -// Weights used in the runtime. -mod weights; - -mod bag_thresholds; - -// Governance configurations. -pub mod governance; -use governance::{ - pallet_custom_origins, AuctionAdmin, FellowshipAdmin, GeneralAdmin, LeaseAdmin, StakingAdmin, - Treasurer, TreasurySpender, -}; - -pub mod xcm_config; - -impl_runtime_weights!(polkadot_runtime_constants); - -// Make the WASM binary available. -#[cfg(feature = "std")] -include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); - -// Polkadot version identifier; -/// Runtime version (Polkadot). -#[sp_version::runtime_version] -pub const VERSION: RuntimeVersion = RuntimeVersion { - spec_name: create_runtime_str!("polkadot"), - impl_name: create_runtime_str!("parity-polkadot"), - authoring_version: 0, - spec_version: 9430, - impl_version: 0, - apis: RUNTIME_API_VERSIONS, - transaction_version: 24, - state_version: 0, -}; - -/// The BABE epoch configuration at genesis. -pub const BABE_GENESIS_EPOCH_CONFIG: babe_primitives::BabeEpochConfiguration = - babe_primitives::BabeEpochConfiguration { - c: PRIMARY_PROBABILITY, - allowed_slots: babe_primitives::AllowedSlots::PrimaryAndSecondaryVRFSlots, - }; - -/// Native version. -#[cfg(any(feature = "std", test))] -pub fn native_version() -> NativeVersion { - NativeVersion { runtime_version: VERSION, can_author_with: Default::default() } -} - -/// A type to identify calls to the Identity pallet. These will be filtered to prevent invocation, -/// locking the state of the pallet and preventing further updates to identities and sub-identities. -/// The locked state will be the genesis state of a new system chain and then removed from the Relay -/// Chain. -pub struct IdentityCalls; -impl Contains for IdentityCalls { - fn contains(c: &RuntimeCall) -> bool { - matches!(c, RuntimeCall::Identity(_)) - } -} - -parameter_types! { - pub const Version: RuntimeVersion = VERSION; - pub const SS58Prefix: u8 = 0; -} - -impl frame_system::Config for Runtime { - type BaseCallFilter = EverythingBut; - type BlockWeights = BlockWeights; - type BlockLength = BlockLength; - type RuntimeOrigin = RuntimeOrigin; - type RuntimeCall = RuntimeCall; - type Nonce = Nonce; - type Hash = Hash; - type Hashing = BlakeTwo256; - type AccountId = AccountId; - type Lookup = AccountIdLookup; - type Block = Block; - type RuntimeEvent = RuntimeEvent; - type BlockHashCount = BlockHashCount; - type DbWeight = RocksDbWeight; - type Version = Version; - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = weights::frame_system::WeightInfo; - type SS58Prefix = SS58Prefix; - type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; -} - -parameter_types! { - pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * - BlockWeights::get().max_block; - pub const MaxScheduledPerBlock: u32 = 50; - pub const NoPreimagePostponement: Option = Some(10); -} - -/// Used the compare the privilege of an origin inside the scheduler. -pub struct OriginPrivilegeCmp; - -impl PrivilegeCmp for OriginPrivilegeCmp { - fn cmp_privilege(left: &OriginCaller, right: &OriginCaller) -> Option { - if left == right { - return Some(Ordering::Equal) - } - - match (left, right) { - // Root is greater than anything. - (OriginCaller::system(frame_system::RawOrigin::Root), _) => Some(Ordering::Greater), - // For every other origin we don't care, as they are not used for `ScheduleOrigin`. - _ => None, - } - } -} - -impl pallet_scheduler::Config for Runtime { - type RuntimeOrigin = RuntimeOrigin; - type RuntimeEvent = RuntimeEvent; - type PalletsOrigin = OriginCaller; - type RuntimeCall = RuntimeCall; - type MaximumWeight = MaximumSchedulerWeight; - // The goal of having ScheduleOrigin include AuctionAdmin is to allow the auctions track of - // OpenGov to schedule periodic auctions. - // Also allow Treasurer to schedule recurring payments. - type ScheduleOrigin = EitherOf, AuctionAdmin>, Treasurer>; - type MaxScheduledPerBlock = MaxScheduledPerBlock; - type WeightInfo = weights::pallet_scheduler::WeightInfo; - type OriginPrivilegeCmp = OriginPrivilegeCmp; - type Preimages = Preimage; -} - -parameter_types! { - pub const PreimageBaseDeposit: Balance = deposit(2, 64); - pub const PreimageByteDeposit: Balance = deposit(0, 1); - pub const PreimageHoldReason: RuntimeHoldReason = RuntimeHoldReason::Preimage(pallet_preimage::HoldReason::Preimage); -} - -impl pallet_preimage::Config for Runtime { - type WeightInfo = weights::pallet_preimage::WeightInfo; - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type ManagerOrigin = EnsureRoot; - type Consideration = HoldConsideration< - AccountId, - Balances, - PreimageHoldReason, - LinearStoragePrice, - >; -} - -parameter_types! { - pub EpochDuration: u64 = prod_or_fast!( - EPOCH_DURATION_IN_SLOTS as u64, - 2 * MINUTES as u64, - "DOT_EPOCH_DURATION" - ); - pub const ExpectedBlockTime: Moment = MILLISECS_PER_BLOCK; - pub ReportLongevity: u64 = - BondingDuration::get() as u64 * SessionsPerEra::get() as u64 * EpochDuration::get(); -} - -impl pallet_babe::Config for Runtime { - type EpochDuration = EpochDuration; - type ExpectedBlockTime = ExpectedBlockTime; - - // session module is the trigger - type EpochChangeTrigger = pallet_babe::ExternalTrigger; - - type DisabledValidators = Session; - - type WeightInfo = (); - - type MaxAuthorities = MaxAuthorities; - type MaxNominators = MaxNominatorRewardedPerValidator; - - type KeyOwnerProof = - >::Proof; - - type EquivocationReportSystem = - pallet_babe::EquivocationReportSystem; -} - -parameter_types! { - pub const IndexDeposit: Balance = 10 * DOLLARS; -} - -impl pallet_indices::Config for Runtime { - type AccountIndex = AccountIndex; - type Currency = Balances; - type Deposit = IndexDeposit; - type RuntimeEvent = RuntimeEvent; - type WeightInfo = weights::pallet_indices::WeightInfo; -} - -parameter_types! { - pub const ExistentialDeposit: Balance = EXISTENTIAL_DEPOSIT; - pub const MaxLocks: u32 = 50; - pub const MaxReserves: u32 = 50; -} - -impl pallet_balances::Config for Runtime { - type Balance = Balance; - type DustRemoval = (); - type RuntimeEvent = RuntimeEvent; - type ExistentialDeposit = ExistentialDeposit; - type AccountStore = System; - type MaxLocks = MaxLocks; - type MaxReserves = MaxReserves; - type ReserveIdentifier = [u8; 8]; - type WeightInfo = weights::pallet_balances::WeightInfo; - type RuntimeHoldReason = RuntimeHoldReason; - type FreezeIdentifier = (); - type MaxHolds = ConstU32<1>; - type MaxFreezes = ConstU32<0>; -} - -parameter_types! { - pub const TransactionByteFee: Balance = 10 * MILLICENTS; - /// This value increases the priority of `Operational` transactions by adding - /// a "virtual tip" that's equal to the `OperationalFeeMultiplier * final_fee`. - pub const OperationalFeeMultiplier: u8 = 5; -} - -impl pallet_transaction_payment::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type OnChargeTransaction = CurrencyAdapter>; - type OperationalFeeMultiplier = OperationalFeeMultiplier; - type WeightToFee = WeightToFee; - type LengthToFee = ConstantMultiplier; - type FeeMultiplierUpdate = SlowAdjustingFeeUpdate; -} - -parameter_types! { - pub const MinimumPeriod: u64 = SLOT_DURATION / 2; -} -impl pallet_timestamp::Config for Runtime { - type Moment = u64; - type OnTimestampSet = Babe; - type MinimumPeriod = MinimumPeriod; - type WeightInfo = weights::pallet_timestamp::WeightInfo; -} - -impl pallet_authorship::Config for Runtime { - type FindAuthor = pallet_session::FindAccountFromAuthorIndex; - type EventHandler = (Staking, ImOnline); -} - -impl_opaque_keys! { - pub struct SessionKeys { - pub grandpa: Grandpa, - pub babe: Babe, - pub im_online: ImOnline, - pub para_validator: Initializer, - pub para_assignment: ParaSessionInfo, - pub authority_discovery: AuthorityDiscovery, - } -} - -impl pallet_session::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type ValidatorId = AccountId; - type ValidatorIdOf = pallet_staking::StashOf; - type ShouldEndSession = Babe; - type NextSessionRotation = Babe; - type SessionManager = pallet_session::historical::NoteHistoricalRoot; - type SessionHandler = ::KeyTypeIdProviders; - type Keys = SessionKeys; - type WeightInfo = weights::pallet_session::WeightInfo; -} - -impl pallet_session::historical::Config for Runtime { - type FullIdentification = pallet_staking::Exposure; - type FullIdentificationOf = pallet_staking::ExposureOf; -} - -parameter_types! { - // phase durations. 1/4 of the last session for each. - // in testing: 1min or half of the session for each - pub SignedPhase: u32 = prod_or_fast!( - EPOCH_DURATION_IN_SLOTS / 4, - (1 * MINUTES).min(EpochDuration::get().saturated_into::() / 2), - "DOT_SIGNED_PHASE" - ); - pub UnsignedPhase: u32 = prod_or_fast!( - EPOCH_DURATION_IN_SLOTS / 4, - (1 * MINUTES).min(EpochDuration::get().saturated_into::() / 2), - "DOT_UNSIGNED_PHASE" - ); - - // signed config - pub const SignedMaxSubmissions: u32 = 16; - pub const SignedMaxRefunds: u32 = 16 / 4; - pub const SignedFixedDeposit: Balance = deposit(2, 0); - pub const SignedDepositIncreaseFactor: Percent = Percent::from_percent(10); - // 40 DOTs fixed deposit.. - pub const SignedDepositBase: Balance = deposit(2, 0); - // 0.01 DOT per KB of solution data. - pub const SignedDepositByte: Balance = deposit(0, 10) / 1024; - // Each good submission will get 1 DOT as reward - pub SignedRewardBase: Balance = 1 * UNITS; - pub BetterUnsignedThreshold: Perbill = Perbill::from_rational(5u32, 10_000); - - // 4 hour session, 1 hour unsigned phase, 32 offchain executions. - pub OffchainRepeat: BlockNumber = UnsignedPhase::get() / 32; - - pub const MaxElectingVoters: u32 = 22_500; - /// We take the top 22500 nominators as electing voters and all of the validators as electable - /// targets. Whilst this is the case, we cannot and shall not increase the size of the - /// validator intentions. - pub ElectionBounds: frame_election_provider_support::bounds::ElectionBounds = - ElectionBoundsBuilder::default().voters_count(MaxElectingVoters::get().into()).build(); - /// Setup election pallet to support maximum winners upto 1200. This will mean Staking Pallet - /// cannot have active validators higher than this count. - pub const MaxActiveValidators: u32 = 1200; -} - -generate_solution_type!( - #[compact] - pub struct NposCompactSolution16::< - VoterIndex = u32, - TargetIndex = u16, - Accuracy = sp_runtime::PerU16, - MaxVoters = MaxElectingVoters, - >(16) -); - -pub struct OnChainSeqPhragmen; -impl onchain::Config for OnChainSeqPhragmen { - type System = Runtime; - type Solver = SequentialPhragmen; - type DataProvider = Staking; - type WeightInfo = weights::frame_election_provider_support::WeightInfo; - type MaxWinners = MaxActiveValidators; - type Bounds = ElectionBounds; -} - -impl pallet_election_provider_multi_phase::MinerConfig for Runtime { - type AccountId = AccountId; - type MaxLength = OffchainSolutionLengthLimit; - type MaxWeight = OffchainSolutionWeightLimit; - type Solution = NposCompactSolution16; - type MaxVotesPerVoter = < - ::DataProvider - as - frame_election_provider_support::ElectionDataProvider - >::MaxVotesPerVoter; - type MaxWinners = MaxActiveValidators; - - // The unsigned submissions have to respect the weight of the submit_unsigned call, thus their - // weight estimate function is wired to this call's weight. - fn solution_weight(v: u32, t: u32, a: u32, d: u32) -> Weight { - < - ::WeightInfo - as - pallet_election_provider_multi_phase::WeightInfo - >::submit_unsigned(v, t, a, d) - } -} - -impl pallet_election_provider_multi_phase::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type EstimateCallFee = TransactionPayment; - type SignedPhase = SignedPhase; - type UnsignedPhase = UnsignedPhase; - type SignedMaxSubmissions = SignedMaxSubmissions; - type SignedMaxRefunds = SignedMaxRefunds; - type SignedRewardBase = SignedRewardBase; - type SignedDepositBase = - GeometricDepositBase; - type SignedDepositByte = SignedDepositByte; - type SignedDepositWeight = (); - type SignedMaxWeight = - ::MaxWeight; - type MinerConfig = Self; - type SlashHandler = (); // burn slashes - type RewardHandler = (); // nothing to do upon rewards - type BetterUnsignedThreshold = BetterUnsignedThreshold; - type BetterSignedThreshold = (); - type OffchainRepeat = OffchainRepeat; - type MinerTxPriority = NposSolutionPriority; - type DataProvider = Staking; - #[cfg(any(feature = "fast-runtime", feature = "runtime-benchmarks"))] - type Fallback = onchain::OnChainExecution; - #[cfg(not(any(feature = "fast-runtime", feature = "runtime-benchmarks")))] - type Fallback = frame_election_provider_support::NoElection<( - AccountId, - BlockNumber, - Staking, - MaxActiveValidators, - )>; - type GovernanceFallback = onchain::OnChainExecution; - type Solver = SequentialPhragmen< - AccountId, - pallet_election_provider_multi_phase::SolutionAccuracyOf, - (), - >; - type BenchmarkingConfig = runtime_common::elections::BenchmarkConfig; - type ForceOrigin = EitherOf, StakingAdmin>; - type WeightInfo = weights::pallet_election_provider_multi_phase::WeightInfo; - type MaxWinners = MaxActiveValidators; - type ElectionBounds = ElectionBounds; -} - -parameter_types! { - pub const BagThresholds: &'static [u64] = &bag_thresholds::THRESHOLDS; -} - -type VoterBagsListInstance = pallet_bags_list::Instance1; -impl pallet_bags_list::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type ScoreProvider = Staking; - type WeightInfo = weights::pallet_bags_list::WeightInfo; - type BagThresholds = BagThresholds; - type Score = sp_npos_elections::VoteWeight; -} - -// TODO #6469: This shouldn't be static, but a lazily cached value, not built unless needed, and -// re-built in case input parameters have changed. The `ideal_stake` should be determined by the -// amount of parachain slots being bid on: this should be around `(75 - 25.min(slots / 4))%`. -pallet_staking_reward_curve::build! { - const REWARD_CURVE: PiecewiseLinear<'static> = curve!( - min_inflation: 0_025_000, - max_inflation: 0_100_000, - // 3:2:1 staked : parachains : float. - // while there's no parachains, then this is 75% staked : 25% float. - ideal_stake: 0_750_000, - falloff: 0_050_000, - max_piece_count: 40, - test_precision: 0_005_000, - ); -} - -parameter_types! { - // Six sessions in an era (24 hours). - pub const SessionsPerEra: SessionIndex = prod_or_fast!(6, 1); - - // 28 eras for unbonding (28 days). - pub BondingDuration: sp_staking::EraIndex = prod_or_fast!( - 28, - 28, - "DOT_BONDING_DURATION" - ); - pub SlashDeferDuration: sp_staking::EraIndex = prod_or_fast!( - 27, - 27, - "DOT_SLASH_DEFER_DURATION" - ); - pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; - pub const MaxNominatorRewardedPerValidator: u32 = 512; - pub const OffendingValidatorsThreshold: Perbill = Perbill::from_percent(17); - // 16 - pub const MaxNominations: u32 = ::LIMIT as u32; -} - -pub struct EraPayout; -impl pallet_staking::EraPayout for EraPayout { - fn era_payout( - total_staked: Balance, - total_issuance: Balance, - era_duration_millis: u64, - ) -> (Balance, Balance) { - // all para-ids that are not active. - let auctioned_slots = Paras::parachains() - .into_iter() - // all active para-ids that do not belong to a system chain is the number - // of parachains that we should take into account for inflation. - .filter(|i| *i >= LOWEST_PUBLIC_ID) - .count() as u64; - - const MAX_ANNUAL_INFLATION: Perquintill = Perquintill::from_percent(10); - const MILLISECONDS_PER_YEAR: u64 = 1000 * 3600 * 24 * 36525 / 100; - - runtime_common::impls::era_payout( - total_staked, - total_issuance, - MAX_ANNUAL_INFLATION, - Perquintill::from_rational(era_duration_millis, MILLISECONDS_PER_YEAR), - auctioned_slots, - ) - } -} - -impl pallet_staking::Config for Runtime { - type Currency = Balances; - type CurrencyBalance = Balance; - type UnixTime = Timestamp; - type CurrencyToVote = CurrencyToVote; - type RewardRemainder = Treasury; - type RuntimeEvent = RuntimeEvent; - type Slash = Treasury; - type Reward = (); - type SessionsPerEra = SessionsPerEra; - type BondingDuration = BondingDuration; - type SlashDeferDuration = SlashDeferDuration; - type AdminOrigin = EitherOf, StakingAdmin>; - type SessionInterface = Self; - type EraPayout = EraPayout; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; - type OffendingValidatorsThreshold = OffendingValidatorsThreshold; - type NextNewSession = Session; - type ElectionProvider = ElectionProviderMultiPhase; - type GenesisElectionProvider = onchain::OnChainExecution; - type VoterList = VoterList; - type TargetList = UseValidatorsMap; - type NominationsQuota = pallet_staking::FixedNominationsQuota<{ MaxNominations::get() }>; - type MaxUnlockingChunks = frame_support::traits::ConstU32<32>; - type HistoryDepth = frame_support::traits::ConstU32<84>; - type BenchmarkingConfig = runtime_common::StakingBenchmarkingConfig; - type EventListeners = NominationPools; - type WeightInfo = weights::pallet_staking::WeightInfo; -} - -impl pallet_fast_unstake::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type BatchSize = frame_support::traits::ConstU32<16>; - type Deposit = frame_support::traits::ConstU128<{ UNITS }>; - type ControlOrigin = EnsureRoot; - type Staking = Staking; - type MaxErasToCheckPerBlock = ConstU32<1>; - #[cfg(feature = "runtime-benchmarks")] - type MaxBackersPerValidator = MaxNominatorRewardedPerValidator; - type WeightInfo = weights::pallet_fast_unstake::WeightInfo; -} - -parameter_types! { - // Minimum 4 CENTS/byte - pub const BasicDeposit: Balance = deposit(1, 258); - pub const FieldDeposit: Balance = deposit(0, 66); - pub const SubAccountDeposit: Balance = deposit(1, 53); - pub const MaxSubAccounts: u32 = 100; - pub const MaxAdditionalFields: u32 = 100; - pub const MaxRegistrars: u32 = 20; -} - -impl pallet_identity::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type BasicDeposit = BasicDeposit; - type FieldDeposit = FieldDeposit; - type SubAccountDeposit = SubAccountDeposit; - type MaxSubAccounts = MaxSubAccounts; - type MaxAdditionalFields = MaxAdditionalFields; - type MaxRegistrars = MaxRegistrars; - type Slashed = Treasury; - type ForceOrigin = EitherOf, GeneralAdmin>; - type RegistrarOrigin = EitherOf, GeneralAdmin>; - type WeightInfo = weights::pallet_identity::WeightInfo; -} - -parameter_types! { - pub const ProposalBond: Permill = Permill::from_percent(5); - pub const ProposalBondMinimum: Balance = 100 * DOLLARS; - pub const ProposalBondMaximum: Balance = 500 * DOLLARS; - pub const SpendPeriod: BlockNumber = 24 * DAYS; - pub const Burn: Permill = Permill::from_percent(1); - pub const TreasuryPalletId: PalletId = PalletId(*b"py/trsry"); - - pub const TipCountdown: BlockNumber = 1 * DAYS; - pub const TipFindersFee: Percent = Percent::from_percent(20); - pub const TipReportDepositBase: Balance = 1 * DOLLARS; - pub const DataDepositPerByte: Balance = 1 * CENTS; - pub const MaxApprovals: u32 = 100; - pub const MaxAuthorities: u32 = 100_000; - pub const MaxKeys: u32 = 10_000; - pub const MaxPeerInHeartbeats: u32 = 10_000; - pub const RootSpendOriginMaxAmount: Balance = Balance::MAX; - pub const CouncilSpendOriginMaxAmount: Balance = Balance::MAX; -} - -impl pallet_treasury::Config for Runtime { - type PalletId = TreasuryPalletId; - type Currency = Balances; - type ApproveOrigin = EitherOfDiverse, Treasurer>; - type RejectOrigin = EitherOfDiverse, Treasurer>; - type RuntimeEvent = RuntimeEvent; - type OnSlash = Treasury; - type ProposalBond = ProposalBond; - type ProposalBondMinimum = ProposalBondMinimum; - type ProposalBondMaximum = ProposalBondMaximum; - type SpendPeriod = SpendPeriod; - type Burn = Burn; - type BurnDestination = (); - type SpendFunds = Bounties; - type MaxApprovals = MaxApprovals; - type WeightInfo = weights::pallet_treasury::WeightInfo; - type SpendOrigin = TreasurySpender; -} - -parameter_types! { - pub const BountyDepositBase: Balance = 1 * DOLLARS; - pub const BountyDepositPayoutDelay: BlockNumber = 8 * DAYS; - pub const BountyUpdatePeriod: BlockNumber = 90 * DAYS; - pub const MaximumReasonLength: u32 = 16384; - pub const CuratorDepositMultiplier: Permill = Permill::from_percent(50); - pub const CuratorDepositMin: Balance = 10 * DOLLARS; - pub const CuratorDepositMax: Balance = 200 * DOLLARS; - pub const BountyValueMinimum: Balance = 10 * DOLLARS; -} - -impl pallet_bounties::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type BountyDepositBase = BountyDepositBase; - type BountyDepositPayoutDelay = BountyDepositPayoutDelay; - type BountyUpdatePeriod = BountyUpdatePeriod; - type CuratorDepositMultiplier = CuratorDepositMultiplier; - type CuratorDepositMin = CuratorDepositMin; - type CuratorDepositMax = CuratorDepositMax; - type BountyValueMinimum = BountyValueMinimum; - type ChildBountyManager = ChildBounties; - type DataDepositPerByte = DataDepositPerByte; - type MaximumReasonLength = MaximumReasonLength; - type WeightInfo = weights::pallet_bounties::WeightInfo; -} - -parameter_types! { - pub const MaxActiveChildBountyCount: u32 = 100; - pub const ChildBountyValueMinimum: Balance = BountyValueMinimum::get() / 10; -} - -impl pallet_child_bounties::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type MaxActiveChildBountyCount = MaxActiveChildBountyCount; - type ChildBountyValueMinimum = ChildBountyValueMinimum; - type WeightInfo = weights::pallet_child_bounties::WeightInfo; -} - -impl pallet_offences::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type IdentificationTuple = pallet_session::historical::IdentificationTuple; - type OnOffenceHandler = Staking; -} - -impl pallet_authority_discovery::Config for Runtime { - type MaxAuthorities = MaxAuthorities; -} - -parameter_types! { - pub NposSolutionPriority: TransactionPriority = - Perbill::from_percent(90) * TransactionPriority::max_value(); - pub const ImOnlineUnsignedPriority: TransactionPriority = TransactionPriority::max_value(); -} - -impl pallet_im_online::Config for Runtime { - type AuthorityId = ImOnlineId; - type RuntimeEvent = RuntimeEvent; - type ValidatorSet = Historical; - type NextSessionRotation = Babe; - type ReportUnresponsiveness = Offences; - type UnsignedPriority = ImOnlineUnsignedPriority; - type WeightInfo = weights::pallet_im_online::WeightInfo; - type MaxKeys = MaxKeys; - type MaxPeerInHeartbeats = MaxPeerInHeartbeats; -} - -parameter_types! { - pub MaxSetIdSessionEntries: u32 = BondingDuration::get() * SessionsPerEra::get(); -} - -impl pallet_grandpa::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - - type WeightInfo = (); - type MaxAuthorities = MaxAuthorities; - type MaxNominators = MaxNominatorRewardedPerValidator; - type MaxSetIdSessionEntries = MaxSetIdSessionEntries; - - type KeyOwnerProof = >::Proof; - - type EquivocationReportSystem = - pallet_grandpa::EquivocationReportSystem; -} - -/// Submits a transaction with the node's public and signature type. Adheres to the signed extension -/// format of the chain. -impl frame_system::offchain::CreateSignedTransaction for Runtime -where - RuntimeCall: From, -{ - fn create_transaction>( - call: RuntimeCall, - public: ::Signer, - account: AccountId, - nonce: ::Nonce, - ) -> Option<(RuntimeCall, ::SignaturePayload)> { - use sp_runtime::traits::StaticLookup; - // take the biggest period possible. - let period = - BlockHashCount::get().checked_next_power_of_two().map(|c| c / 2).unwrap_or(2) as u64; - - let current_block = System::block_number() - .saturated_into::() - // The `System::block_number` is initialized with `n+1`, - // so the actual block number is `n`. - .saturating_sub(1); - let tip = 0; - let extra: SignedExtra = ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(generic::Era::mortal( - period, - current_block, - )), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - claims::PrevalidateAttests::::new(), - ); - let raw_payload = SignedPayload::new(call, extra) - .map_err(|e| { - log::warn!("Unable to create signed payload: {:?}", e); - }) - .ok()?; - let signature = raw_payload.using_encoded(|payload| C::sign(payload, public))?; - let (call, extra, _) = raw_payload.deconstruct(); - let address = ::Lookup::unlookup(account); - Some((call, (address, signature, extra))) - } -} - -impl frame_system::offchain::SigningTypes for Runtime { - type Public = ::Signer; - type Signature = Signature; -} - -impl frame_system::offchain::SendTransactionTypes for Runtime -where - RuntimeCall: From, -{ - type Extrinsic = UncheckedExtrinsic; - type OverarchingCall = RuntimeCall; -} - -parameter_types! { - // Deposit for a parathread (on-demand parachain) - pub const ParathreadDeposit: Balance = 500 * DOLLARS; - pub const MaxRetries: u32 = 3; -} - -parameter_types! { - pub Prefix: &'static [u8] = b"Pay DOTs to the Polkadot account:"; -} - -impl claims::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type VestingSchedule = Vesting; - type Prefix = Prefix; - /// Only Root can move a claim. - type MoveClaimOrigin = EnsureRoot; - type WeightInfo = weights::runtime_common_claims::WeightInfo; -} - -parameter_types! { - pub const MinVestedTransfer: Balance = 1 * DOLLARS; - pub UnvestedFundsAllowedWithdrawReasons: WithdrawReasons = - WithdrawReasons::except(WithdrawReasons::TRANSFER | WithdrawReasons::RESERVE); -} - -impl pallet_vesting::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type BlockNumberToBalance = ConvertInto; - type MinVestedTransfer = MinVestedTransfer; - type WeightInfo = weights::pallet_vesting::WeightInfo; - type UnvestedFundsAllowedWithdrawReasons = UnvestedFundsAllowedWithdrawReasons; - const MAX_VESTING_SCHEDULES: u32 = 28; -} - -impl pallet_utility::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type RuntimeCall = RuntimeCall; - type PalletsOrigin = OriginCaller; - type WeightInfo = weights::pallet_utility::WeightInfo; -} - -parameter_types! { - // One storage item; key size is 32; value is size 4+4+16+32 bytes = 56 bytes. - pub const DepositBase: Balance = deposit(1, 88); - // Additional storage item size of 32 bytes. - pub const DepositFactor: Balance = deposit(0, 32); - pub const MaxSignatories: u32 = 100; -} - -impl pallet_multisig::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type RuntimeCall = RuntimeCall; - type Currency = Balances; - type DepositBase = DepositBase; - type DepositFactor = DepositFactor; - type MaxSignatories = MaxSignatories; - type WeightInfo = weights::pallet_multisig::WeightInfo; -} - -parameter_types! { - // One storage item; key size 32, value size 8; . - pub const ProxyDepositBase: Balance = deposit(1, 8); - // Additional storage item size of 33 bytes. - pub const ProxyDepositFactor: Balance = deposit(0, 33); - pub const MaxProxies: u16 = 32; - pub const AnnouncementDepositBase: Balance = deposit(1, 8); - pub const AnnouncementDepositFactor: Balance = deposit(0, 66); - pub const MaxPending: u16 = 32; -} - -/// The type used to represent the kinds of proxying allowed. -#[derive( - Copy, - Clone, - Eq, - PartialEq, - Ord, - PartialOrd, - Encode, - Decode, - RuntimeDebug, - MaxEncodedLen, - scale_info::TypeInfo, -)] -pub enum ProxyType { - Any = 0, - NonTransfer = 1, - Governance = 2, - Staking = 3, - // Skip 4 as it is now removed (was SudoBalances) - IdentityJudgement = 5, - CancelProxy = 6, - Auction = 7, - NominationPools = 8, -} - -#[cfg(test)] -mod proxy_type_tests { - use super::*; - - #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, RuntimeDebug)] - pub enum OldProxyType { - Any, - NonTransfer, - Governance, - Staking, - SudoBalances, - IdentityJudgement, - } - - #[test] - fn proxy_type_decodes_correctly() { - for (i, j) in vec![ - (OldProxyType::Any, ProxyType::Any), - (OldProxyType::NonTransfer, ProxyType::NonTransfer), - (OldProxyType::Governance, ProxyType::Governance), - (OldProxyType::Staking, ProxyType::Staking), - (OldProxyType::IdentityJudgement, ProxyType::IdentityJudgement), - ] - .into_iter() - { - assert_eq!(i.encode(), j.encode()); - } - assert!(ProxyType::decode(&mut &OldProxyType::SudoBalances.encode()[..]).is_err()); - } -} - -impl Default for ProxyType { - fn default() -> Self { - Self::Any - } -} -impl InstanceFilter for ProxyType { - fn filter(&self, c: &RuntimeCall) -> bool { - match self { - ProxyType::Any => true, - ProxyType::NonTransfer => matches!( - c, - RuntimeCall::System(..) | - RuntimeCall::Scheduler(..) | - RuntimeCall::Babe(..) | - RuntimeCall::Timestamp(..) | - RuntimeCall::Indices(pallet_indices::Call::claim{..}) | - RuntimeCall::Indices(pallet_indices::Call::free{..}) | - RuntimeCall::Indices(pallet_indices::Call::freeze{..}) | - // Specifically omitting Indices `transfer`, `force_transfer` - // Specifically omitting the entire Balances pallet - RuntimeCall::Staking(..) | - RuntimeCall::Session(..) | - RuntimeCall::Grandpa(..) | - RuntimeCall::ImOnline(..) | - RuntimeCall::Treasury(..) | - RuntimeCall::Bounties(..) | - RuntimeCall::ChildBounties(..) | - RuntimeCall::ConvictionVoting(..) | - RuntimeCall::Referenda(..) | - RuntimeCall::Whitelist(..) | - RuntimeCall::Claims(..) | - RuntimeCall::Vesting(pallet_vesting::Call::vest{..}) | - RuntimeCall::Vesting(pallet_vesting::Call::vest_other{..}) | - // Specifically omitting Vesting `vested_transfer`, and `force_vested_transfer` - RuntimeCall::Utility(..) | - RuntimeCall::Identity(..) | - RuntimeCall::Proxy(..) | - RuntimeCall::Multisig(..) | - RuntimeCall::Registrar(paras_registrar::Call::register {..}) | - RuntimeCall::Registrar(paras_registrar::Call::deregister {..}) | - // Specifically omitting Registrar `swap` - RuntimeCall::Registrar(paras_registrar::Call::reserve {..}) | - RuntimeCall::Crowdloan(..) | - RuntimeCall::Slots(..) | - RuntimeCall::Auctions(..) | // Specifically omitting the entire XCM Pallet - RuntimeCall::VoterList(..) | - RuntimeCall::NominationPools(..) | - RuntimeCall::FastUnstake(..) - ), - ProxyType::Governance => matches!( - c, - RuntimeCall::Treasury(..) | - RuntimeCall::Bounties(..) | - RuntimeCall::Utility(..) | - RuntimeCall::ChildBounties(..) | - RuntimeCall::ConvictionVoting(..) | - RuntimeCall::Referenda(..) | - RuntimeCall::Whitelist(..) - ), - ProxyType::Staking => { - matches!( - c, - RuntimeCall::Staking(..) | - RuntimeCall::Session(..) | RuntimeCall::Utility(..) | - RuntimeCall::FastUnstake(..) | - RuntimeCall::VoterList(..) | - RuntimeCall::NominationPools(..) - ) - }, - ProxyType::NominationPools => { - matches!(c, RuntimeCall::NominationPools(..) | RuntimeCall::Utility(..)) - }, - ProxyType::IdentityJudgement => matches!( - c, - RuntimeCall::Identity(pallet_identity::Call::provide_judgement { .. }) | - RuntimeCall::Utility(..) - ), - ProxyType::CancelProxy => { - matches!(c, RuntimeCall::Proxy(pallet_proxy::Call::reject_announcement { .. })) - }, - ProxyType::Auction => matches!( - c, - RuntimeCall::Auctions(..) | - RuntimeCall::Crowdloan(..) | - RuntimeCall::Registrar(..) | - RuntimeCall::Slots(..) - ), - } - } - fn is_superset(&self, o: &Self) -> bool { - match (self, o) { - (x, y) if x == y => true, - (ProxyType::Any, _) => true, - (_, ProxyType::Any) => false, - (ProxyType::NonTransfer, _) => true, - _ => false, - } - } -} - -impl pallet_proxy::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type RuntimeCall = RuntimeCall; - type Currency = Balances; - type ProxyType = ProxyType; - type ProxyDepositBase = ProxyDepositBase; - type ProxyDepositFactor = ProxyDepositFactor; - type MaxProxies = MaxProxies; - type WeightInfo = weights::pallet_proxy::WeightInfo; - type MaxPending = MaxPending; - type CallHasher = BlakeTwo256; - type AnnouncementDepositBase = AnnouncementDepositBase; - type AnnouncementDepositFactor = AnnouncementDepositFactor; -} - -impl parachains_origin::Config for Runtime {} - -impl parachains_configuration::Config for Runtime { - type WeightInfo = weights::runtime_parachains_configuration::WeightInfo; -} - -impl parachains_shared::Config for Runtime {} - -impl parachains_session_info::Config for Runtime { - type ValidatorSet = Historical; -} - -impl parachains_inclusion::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type DisputesHandler = ParasDisputes; - type RewardValidators = parachains_reward_points::RewardValidatorsWithEraPoints; - type MessageQueue = MessageQueue; - type WeightInfo = weights::runtime_parachains_inclusion::WeightInfo; -} - -parameter_types! { - pub const ParasUnsignedPriority: TransactionPriority = TransactionPriority::max_value(); -} - -impl parachains_paras::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type WeightInfo = weights::runtime_parachains_paras::WeightInfo; - type UnsignedPriority = ParasUnsignedPriority; - type QueueFootprinter = ParaInclusion; - type NextSessionRotation = Babe; - type OnNewHead = Registrar; -} - -parameter_types! { - /// Amount of weight that can be spent per block to service messages. - /// - /// # WARNING - /// - /// This is not a good value for para-chains since the `Scheduler` already uses up to 80% block weight. - pub MessageQueueServiceWeight: Weight = Perbill::from_percent(20) * BlockWeights::get().max_block; - pub const MessageQueueHeapSize: u32 = 65_536; - pub const MessageQueueMaxStale: u32 = 8; -} - -/// Message processor to handle any messages that were enqueued into the `MessageQueue` pallet. -pub struct MessageProcessor; -impl ProcessMessage for MessageProcessor { - type Origin = AggregateMessageOrigin; - - fn process_message( - message: &[u8], - origin: Self::Origin, - meter: &mut WeightMeter, - id: &mut [u8; 32], - ) -> Result { - let para = match origin { - AggregateMessageOrigin::Ump(UmpQueueId::Para(para)) => para, - }; - xcm_builder::ProcessXcmMessage::< - Junction, - xcm_executor::XcmExecutor, - RuntimeCall, - >::process_message(message, Junction::Parachain(para.into()), meter, id) - } -} - -impl pallet_message_queue::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Size = u32; - type HeapSize = MessageQueueHeapSize; - type MaxStale = MessageQueueMaxStale; - type ServiceWeight = MessageQueueServiceWeight; - #[cfg(not(feature = "runtime-benchmarks"))] - type MessageProcessor = MessageProcessor; - #[cfg(feature = "runtime-benchmarks")] - type MessageProcessor = - pallet_message_queue::mock_helpers::NoopMessageProcessor; - type QueueChangeHandler = ParaInclusion; - type QueuePausedQuery = (); - type WeightInfo = weights::pallet_message_queue::WeightInfo; -} - -impl parachains_dmp::Config for Runtime {} - -impl parachains_hrmp::Config for Runtime { - type RuntimeOrigin = RuntimeOrigin; - type RuntimeEvent = RuntimeEvent; - type ChannelManager = EitherOf, GeneralAdmin>; - type Currency = Balances; - type WeightInfo = weights::runtime_parachains_hrmp::WeightInfo; -} - -impl parachains_paras_inherent::Config for Runtime { - type WeightInfo = weights::runtime_parachains_paras_inherent::WeightInfo; -} - -impl parachains_scheduler::Config for Runtime { - type AssignmentProvider = ParaAssignmentProvider; -} - -impl parachains_assigner_parachains::Config for Runtime {} - -impl parachains_initializer::Config for Runtime { - type Randomness = pallet_babe::RandomnessFromOneEpochAgo; - type ForceOrigin = EnsureRoot; - type WeightInfo = weights::runtime_parachains_initializer::WeightInfo; -} - -impl parachains_disputes::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type RewardValidators = parachains_reward_points::RewardValidatorsWithEraPoints; - type SlashingHandler = parachains_slashing::SlashValidatorsForDisputes; - type WeightInfo = weights::runtime_parachains_disputes::WeightInfo; -} - -impl parachains_slashing::Config for Runtime { - type KeyOwnerProofSystem = Historical; - type KeyOwnerProof = - >::Proof; - type KeyOwnerIdentification = >::IdentificationTuple; - type HandleReports = parachains_slashing::SlashingReportHandler< - Self::KeyOwnerIdentification, - Offences, - ReportLongevity, - >; - type WeightInfo = weights::runtime_parachains_disputes_slashing::WeightInfo; - type BenchmarkingConfig = parachains_slashing::BenchConfig<1000>; -} - -parameter_types! { - // Mostly arbitrary deposit price, but should provide an adequate incentive not to spam reserve - // `ParaId`s. - pub const ParaDeposit: Balance = 100 * DOLLARS; - pub const ParaDataByteDeposit: Balance = deposit(0, 1); -} - -impl paras_registrar::Config for Runtime { - type RuntimeOrigin = RuntimeOrigin; - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type OnSwap = (Crowdloan, Slots); - type ParaDeposit = ParaDeposit; - type DataDepositPerByte = ParaDataByteDeposit; - type WeightInfo = weights::runtime_common_paras_registrar::WeightInfo; -} - -parameter_types! { - // 12 weeks = 3 months per lease period -> 8 lease periods ~ 2 years - pub LeasePeriod: BlockNumber = prod_or_fast!(12 * WEEKS, 12 * WEEKS, "DOT_LEASE_PERIOD"); - // Polkadot Genesis was on May 26, 2020. - // Target Parachain Onboarding Date: Dec 15, 2021. - // Difference is 568 days. - // We want a lease period to start on the target onboarding date. - // 568 % (12 * 7) = 64 day offset - pub LeaseOffset: BlockNumber = prod_or_fast!(64 * DAYS, 0, "DOT_LEASE_OFFSET"); -} - -impl slots::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type Registrar = Registrar; - type LeasePeriod = LeasePeriod; - type LeaseOffset = LeaseOffset; - type ForceOrigin = EitherOf, LeaseAdmin>; - type WeightInfo = weights::runtime_common_slots::WeightInfo; -} - -parameter_types! { - pub const CrowdloanId: PalletId = PalletId(*b"py/cfund"); - // Accounts for 10_000 contributions, each using 48 bytes (16 bytes for balance, and 32 bytes - // for a memo). - pub const SubmissionDeposit: Balance = deposit(1, 480_000); - // The minimum crowdloan contribution. - pub const MinContribution: Balance = 5 * DOLLARS; - pub const RemoveKeysLimit: u32 = 1000; - // Allow 32 bytes for an additional memo to a crowdloan. - pub const MaxMemoLength: u8 = 32; -} - -impl crowdloan::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type PalletId = CrowdloanId; - type SubmissionDeposit = SubmissionDeposit; - type MinContribution = MinContribution; - type RemoveKeysLimit = RemoveKeysLimit; - type Registrar = Registrar; - type Auctioneer = Auctions; - type MaxMemoLength = MaxMemoLength; - type WeightInfo = weights::runtime_common_crowdloan::WeightInfo; -} - -parameter_types! { - // The average auction is 7 days long, so this will be 70% for ending period. - // 5 Days = 72000 Blocks @ 6 sec per block - pub const EndingPeriod: BlockNumber = 5 * DAYS; - // ~ 1000 samples per day -> ~ 20 blocks per sample -> 2 minute samples - pub const SampleLength: BlockNumber = 2 * MINUTES; -} - -impl auctions::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Leaser = Slots; - type Registrar = Registrar; - type EndingPeriod = EndingPeriod; - type SampleLength = SampleLength; - type Randomness = pallet_babe::RandomnessFromOneEpochAgo; - type InitiateOrigin = EitherOf, AuctionAdmin>; - type WeightInfo = weights::runtime_common_auctions::WeightInfo; -} - -parameter_types! { - pub const PoolsPalletId: PalletId = PalletId(*b"py/nopls"); - // Allow pools that got slashed up to 90% to remain operational. - pub const MaxPointsToBalance: u8 = 10; -} - -impl pallet_nomination_pools::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type Currency = Balances; - type RewardCounter = FixedU128; - type BalanceToU256 = runtime_common::BalanceToU256; - type U256ToBalance = runtime_common::U256ToBalance; - type Staking = Staking; - type PostUnbondingPoolsWindow = frame_support::traits::ConstU32<4>; - type MaxMetadataLen = frame_support::traits::ConstU32<256>; - // we use the same number of allowed unlocking chunks as with staking. - type MaxUnbonding = ::MaxUnlockingChunks; - type PalletId = PoolsPalletId; - type MaxPointsToBalance = MaxPointsToBalance; - type WeightInfo = weights::pallet_nomination_pools::WeightInfo; -} - -pub struct InitiateNominationPools; -impl frame_support::traits::OnRuntimeUpgrade for InitiateNominationPools { - fn on_runtime_upgrade() -> frame_support::weights::Weight { - // we use one as an indicator if this has already been set. - if pallet_nomination_pools::MaxPools::::get().is_none() { - // 5 DOT to join a pool. - pallet_nomination_pools::MinJoinBond::::put(5 * UNITS); - // 100 DOT to create a pool. - pallet_nomination_pools::MinCreateBond::::put(100 * UNITS); - - // Initialize with limits for now. - pallet_nomination_pools::MaxPools::::put(0); - pallet_nomination_pools::MaxPoolMembersPerPool::::put(0); - pallet_nomination_pools::MaxPoolMembers::::put(0); - - log::info!(target: "runtime::polkadot", "pools config initiated 🎉"); - ::DbWeight::get().reads_writes(1, 5) - } else { - log::info!(target: "runtime::polkadot", "pools config already initiated 😏"); - ::DbWeight::get().reads(1) - } - } -} - -construct_runtime! { - pub enum Runtime - { - // Basic stuff; balances is uncallable initially. - System: frame_system::{Pallet, Call, Storage, Config, Event} = 0, - Scheduler: pallet_scheduler::{Pallet, Call, Storage, Event} = 1, - Preimage: pallet_preimage::{Pallet, Call, Storage, Event, HoldReason} = 10, - - // Babe must be before session. - Babe: pallet_babe::{Pallet, Call, Storage, Config, ValidateUnsigned} = 2, - - Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent} = 3, - Indices: pallet_indices::{Pallet, Call, Storage, Config, Event} = 4, - Balances: pallet_balances::{Pallet, Call, Storage, Config, Event} = 5, - TransactionPayment: pallet_transaction_payment::{Pallet, Storage, Event} = 32, - - // Consensus support. - // Authorship must be before session in order to note author in the correct session and era - // for im-online and staking. - Authorship: pallet_authorship::{Pallet, Storage} = 6, - Staking: pallet_staking::{Pallet, Call, Storage, Config, Event} = 7, - Offences: pallet_offences::{Pallet, Storage, Event} = 8, - Historical: session_historical::{Pallet} = 33, - Session: pallet_session::{Pallet, Call, Storage, Event, Config} = 9, - Grandpa: pallet_grandpa::{Pallet, Call, Storage, Config, Event, ValidateUnsigned} = 11, - ImOnline: pallet_im_online::{Pallet, Call, Storage, Event, ValidateUnsigned, Config} = 12, - AuthorityDiscovery: pallet_authority_discovery::{Pallet, Config} = 13, - - // OpenGov stuff. - Treasury: pallet_treasury::{Pallet, Call, Storage, Config, Event} = 19, - ConvictionVoting: pallet_conviction_voting::{Pallet, Call, Storage, Event} = 20, - Referenda: pallet_referenda::{Pallet, Call, Storage, Event} = 21, - Origins: pallet_custom_origins::{Origin} = 22, - Whitelist: pallet_whitelist::{Pallet, Call, Storage, Event} = 23, - - // Claims. Usable initially. - Claims: claims::{Pallet, Call, Storage, Event, Config, ValidateUnsigned} = 24, - // Vesting. Usable initially, but removed once all vesting is finished. - Vesting: pallet_vesting::{Pallet, Call, Storage, Event, Config} = 25, - // Cunning utilities. Usable initially. - Utility: pallet_utility::{Pallet, Call, Event} = 26, - - // Identity. Late addition. - Identity: pallet_identity::{Pallet, Call, Storage, Event} = 28, - - // Proxy module. Late addition. - Proxy: pallet_proxy::{Pallet, Call, Storage, Event} = 29, - - // Multisig dispatch. Late addition. - Multisig: pallet_multisig::{Pallet, Call, Storage, Event} = 30, - - // Bounties modules. - Bounties: pallet_bounties::{Pallet, Call, Storage, Event} = 34, - ChildBounties: pallet_child_bounties = 38, - - // Election pallet. Only works with staking, but placed here to maintain indices. - ElectionProviderMultiPhase: pallet_election_provider_multi_phase::{Pallet, Call, Storage, Event, ValidateUnsigned} = 36, - - // Provides a semi-sorted list of nominators for staking. - VoterList: pallet_bags_list::::{Pallet, Call, Storage, Event} = 37, - - // Nomination pools: extension to staking. - NominationPools: pallet_nomination_pools::{Pallet, Call, Storage, Event, Config} = 39, - - // Fast unstake pallet: extension to staking. - FastUnstake: pallet_fast_unstake = 40, - - // Parachains pallets. Start indices at 50 to leave room. - ParachainsOrigin: parachains_origin::{Pallet, Origin} = 50, - Configuration: parachains_configuration::{Pallet, Call, Storage, Config} = 51, - ParasShared: parachains_shared::{Pallet, Call, Storage} = 52, - ParaInclusion: parachains_inclusion::{Pallet, Call, Storage, Event} = 53, - ParaInherent: parachains_paras_inherent::{Pallet, Call, Storage, Inherent} = 54, - ParaScheduler: parachains_scheduler::{Pallet, Storage} = 55, - Paras: parachains_paras::{Pallet, Call, Storage, Event, Config, ValidateUnsigned} = 56, - Initializer: parachains_initializer::{Pallet, Call, Storage} = 57, - Dmp: parachains_dmp::{Pallet, Storage} = 58, - // Ump 59 - Hrmp: parachains_hrmp::{Pallet, Call, Storage, Event, Config} = 60, - ParaSessionInfo: parachains_session_info::{Pallet, Storage} = 61, - ParasDisputes: parachains_disputes::{Pallet, Call, Storage, Event} = 62, - ParasSlashing: parachains_slashing::{Pallet, Call, Storage, ValidateUnsigned} = 63, - ParaAssignmentProvider: parachains_assigner_parachains::{Pallet} = 64, - - // Parachain Onboarding Pallets. Start indices at 70 to leave room. - Registrar: paras_registrar::{Pallet, Call, Storage, Event} = 70, - Slots: slots::{Pallet, Call, Storage, Event} = 71, - Auctions: auctions::{Pallet, Call, Storage, Event} = 72, - Crowdloan: crowdloan::{Pallet, Call, Storage, Event} = 73, - - // Pallet for sending XCM. - XcmPallet: pallet_xcm::{Pallet, Call, Storage, Event, Origin, Config} = 99, - - // Generalized message queue - MessageQueue: pallet_message_queue::{Pallet, Call, Storage, Event} = 100, - } -} - -/// The address format for describing accounts. -pub type Address = sp_runtime::MultiAddress; -/// Block header type as expected by this runtime. -pub type Header = generic::Header; -/// Block type as expected by this runtime. -pub type Block = generic::Block; -/// A Block signed with a Justification -pub type SignedBlock = generic::SignedBlock; -/// `BlockId` type as expected by this runtime. -pub type BlockId = generic::BlockId; -/// The `SignedExtension` to the basic transaction logic. -pub type SignedExtra = ( - frame_system::CheckNonZeroSender, - frame_system::CheckSpecVersion, - frame_system::CheckTxVersion, - frame_system::CheckGenesis, - frame_system::CheckMortality, - frame_system::CheckNonce, - frame_system::CheckWeight, - pallet_transaction_payment::ChargeTransactionPayment, - claims::PrevalidateAttests, -); - -pub struct NominationPoolsMigrationV4OldPallet; -impl Get for NominationPoolsMigrationV4OldPallet { - fn get() -> Perbill { - Perbill::zero() - } -} - -/// All migrations that will run on the next runtime upgrade. -/// -/// This contains the combined migrations of the last 10 releases. It allows to skip runtime -/// upgrades in case governance decides to do so. THE ORDER IS IMPORTANT. -pub type Migrations = migrations::Unreleased; - -/// The runtime migrations per release. -#[allow(deprecated, missing_docs)] -pub mod migrations { - use super::*; - use frame_support::traits::LockIdentifier; - use frame_system::pallet_prelude::BlockNumberFor; - - parameter_types! { - pub const DemocracyPalletName: &'static str = "Democracy"; - pub const CouncilPalletName: &'static str = "Council"; - pub const TechnicalCommitteePalletName: &'static str = "TechnicalCommittee"; - pub const PhragmenElectionPalletName: &'static str = "PhragmenElection"; - pub const TechnicalMembershipPalletName: &'static str = "TechnicalMembership"; - pub const TipsPalletName: &'static str = "Tips"; - pub const PhragmenElectionPalletId: LockIdentifier = *b"phrelect"; - } - - // Special Config for Gov V1 pallets, allowing us to run migrations for them without - // implementing their configs on [`Runtime`]. - pub struct UnlockConfig; - impl pallet_democracy::migrations::unlock_and_unreserve_all_funds::UnlockConfig for UnlockConfig { - type Currency = Balances; - type MaxVotes = ConstU32<100>; - type MaxDeposits = ConstU32<100>; - type AccountId = AccountId; - type BlockNumber = BlockNumberFor; - type DbWeight = ::DbWeight; - type PalletName = DemocracyPalletName; - } - impl pallet_elections_phragmen::migrations::unlock_and_unreserve_all_funds::UnlockConfig - for UnlockConfig - { - type Currency = Balances; - type MaxVotesPerVoter = ConstU32<16>; - type PalletId = PhragmenElectionPalletId; - type AccountId = AccountId; - type DbWeight = ::DbWeight; - type PalletName = PhragmenElectionPalletName; - } - impl pallet_tips::migrations::unreserve_deposits::UnlockConfig<()> for UnlockConfig { - type Currency = Balances; - type Hash = Hash; - type DataDepositPerByte = DataDepositPerByte; - type TipReportDepositBase = TipReportDepositBase; - type AccountId = AccountId; - type BlockNumber = BlockNumberFor; - type DbWeight = ::DbWeight; - type PalletName = TipsPalletName; - } - - pub struct ParachainsToUnlock; - impl Contains for ParachainsToUnlock { - fn contains(id: &ParaId) -> bool { - let id: u32 = (*id).into(); - // polkadot parachains/parathreads that are locked and never produced block - match id { - 2003 | 2015 | 2017 | 2018 | 2025 | 2028 | 2036 | 2038 | 2053 | 2055 | 2090 | - 2097 | 2106 | 3336 | 3338 | 3342 => true, - _ => false, - } - } - } - - /// Unreleased migrations. Add new ones here: - pub type Unreleased = ( - pallet_im_online::migration::v1::Migration, - parachains_configuration::migration::v7::MigrateToV7, - parachains_scheduler::migration::v1::MigrateToV1, - parachains_configuration::migration::v8::MigrateToV8, - - // Gov v1 storage migrations - // https://github.com/paritytech/polkadot/issues/6749 - pallet_elections_phragmen::migrations::unlock_and_unreserve_all_funds::UnlockAndUnreserveAllFunds, - pallet_democracy::migrations::unlock_and_unreserve_all_funds::UnlockAndUnreserveAllFunds, - pallet_tips::migrations::unreserve_deposits::UnreserveDeposits, - - // Delete all Gov v1 pallet storage key/values. - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - frame_support::migrations::RemovePallet::DbWeight>, - - parachains_configuration::migration::v9::MigrateToV9, - // Migrate parachain info format - paras_registrar::migration::VersionCheckedMigrateToV1, - parachains_configuration::migration::v10::MigrateToV10, - ); -} - -/// Unchecked extrinsic type as expected by this runtime. -pub type UncheckedExtrinsic = - generic::UncheckedExtrinsic; -/// Executive: handles dispatch to the various modules. -pub type Executive = frame_executive::Executive< - Runtime, - Block, - frame_system::ChainContext, - Runtime, - AllPalletsWithSystem, - Migrations, ->; - -/// The payload being signed in transactions. -pub type SignedPayload = generic::SignedPayload; - -#[cfg(feature = "runtime-benchmarks")] -mod benches { - frame_benchmarking::define_benchmarks!( - // Polkadot - // NOTE: Make sure to prefix these with `runtime_common::` so - // the that path resolves correctly in the generated file. - [runtime_common::auctions, Auctions] - [runtime_common::claims, Claims] - [runtime_common::crowdloan, Crowdloan] - [runtime_common::slots, Slots] - [runtime_common::paras_registrar, Registrar] - [runtime_parachains::configuration, Configuration] - [runtime_parachains::disputes, ParasDisputes] - [runtime_parachains::disputes::slashing, ParasSlashing] - [runtime_parachains::hrmp, Hrmp] - [runtime_parachains::inclusion, ParaInclusion] - [runtime_parachains::initializer, Initializer] - [runtime_parachains::paras, Paras] - [runtime_parachains::paras_inherent, ParaInherent] - // Substrate - [pallet_bags_list, VoterList] - [pallet_balances, Balances] - [frame_benchmarking::baseline, Baseline::] - [pallet_bounties, Bounties] - [pallet_child_bounties, ChildBounties] - [pallet_election_provider_multi_phase, ElectionProviderMultiPhase] - [frame_election_provider_support, ElectionProviderBench::] - [pallet_fast_unstake, FastUnstake] - [pallet_identity, Identity] - [pallet_im_online, ImOnline] - [pallet_indices, Indices] - [pallet_message_queue, MessageQueue] - [pallet_multisig, Multisig] - [pallet_nomination_pools, NominationPoolsBench::] - [pallet_offences, OffencesBench::] - [pallet_preimage, Preimage] - [pallet_proxy, Proxy] - [pallet_scheduler, Scheduler] - [pallet_session, SessionBench::] - [pallet_staking, Staking] - [frame_system, SystemBench::] - [pallet_timestamp, Timestamp] - [pallet_treasury, Treasury] - [pallet_utility, Utility] - [pallet_vesting, Vesting] - [pallet_conviction_voting, ConvictionVoting] - [pallet_referenda, Referenda] - [pallet_whitelist, Whitelist] - // XCM - [pallet_xcm, XcmPallet] - [pallet_xcm_benchmarks::fungible, pallet_xcm_benchmarks::fungible::Pallet::] - [pallet_xcm_benchmarks::generic, pallet_xcm_benchmarks::generic::Pallet::] - ); -} - -sp_api::impl_runtime_apis! { - impl sp_api::Core for Runtime { - fn version() -> RuntimeVersion { - VERSION - } - - fn execute_block(block: Block) { - Executive::execute_block(block); - } - - fn initialize_block(header: &::Header) { - Executive::initialize_block(header) - } - } - - impl sp_api::Metadata for Runtime { - fn metadata() -> OpaqueMetadata { - OpaqueMetadata::new(Runtime::metadata().into()) - } - - fn metadata_at_version(version: u32) -> Option { - Runtime::metadata_at_version(version) - } - - fn metadata_versions() -> sp_std::vec::Vec { - Runtime::metadata_versions() - } - } - - impl block_builder_api::BlockBuilder for Runtime { - fn apply_extrinsic(extrinsic: ::Extrinsic) -> ApplyExtrinsicResult { - Executive::apply_extrinsic(extrinsic) - } - - fn finalize_block() -> ::Header { - Executive::finalize_block() - } - - fn inherent_extrinsics(data: inherents::InherentData) -> Vec<::Extrinsic> { - data.create_extrinsics() - } - - fn check_inherents( - block: Block, - data: inherents::InherentData, - ) -> inherents::CheckInherentsResult { - data.check_extrinsics(&block) - } - } - - impl pallet_nomination_pools_runtime_api::NominationPoolsApi< - Block, - AccountId, - Balance, - > for Runtime { - fn pending_rewards(member: AccountId) -> Balance { - NominationPools::api_pending_rewards(member).unwrap_or_default() - } - - fn points_to_balance(pool_id: pallet_nomination_pools::PoolId, points: Balance) -> Balance { - NominationPools::api_points_to_balance(pool_id, points) - } - - fn balance_to_points(pool_id: pallet_nomination_pools::PoolId, new_funds: Balance) -> Balance { - NominationPools::api_balance_to_points(pool_id, new_funds) - } - } - - impl pallet_staking_runtime_api::StakingApi for Runtime { - fn nominations_quota(balance: Balance) -> u32 { - Staking::api_nominations_quota(balance) - } - } - - impl tx_pool_api::runtime_api::TaggedTransactionQueue for Runtime { - fn validate_transaction( - source: TransactionSource, - tx: ::Extrinsic, - block_hash: ::Hash, - ) -> TransactionValidity { - Executive::validate_transaction(source, tx, block_hash) - } - } - - impl offchain_primitives::OffchainWorkerApi for Runtime { - fn offchain_worker(header: &::Header) { - Executive::offchain_worker(header) - } - } - - impl primitives::runtime_api::ParachainHost for Runtime { - fn validators() -> Vec { - parachains_runtime_api_impl::validators::() - } - - fn validator_groups() -> (Vec>, GroupRotationInfo) { - parachains_runtime_api_impl::validator_groups::() - } - - fn availability_cores() -> Vec> { - parachains_runtime_api_impl::availability_cores::() - } - - fn persisted_validation_data(para_id: ParaId, assumption: OccupiedCoreAssumption) - -> Option> { - parachains_runtime_api_impl::persisted_validation_data::(para_id, assumption) - } - - fn assumed_validation_data( - para_id: ParaId, - expected_persisted_validation_data_hash: Hash, - ) -> Option<(PersistedValidationData, ValidationCodeHash)> { - parachains_runtime_api_impl::assumed_validation_data::( - para_id, - expected_persisted_validation_data_hash, - ) - } - - fn check_validation_outputs( - para_id: ParaId, - outputs: primitives::CandidateCommitments, - ) -> bool { - parachains_runtime_api_impl::check_validation_outputs::(para_id, outputs) - } - - fn session_index_for_child() -> SessionIndex { - parachains_runtime_api_impl::session_index_for_child::() - } - - fn validation_code(para_id: ParaId, assumption: OccupiedCoreAssumption) - -> Option { - parachains_runtime_api_impl::validation_code::(para_id, assumption) - } - - fn candidate_pending_availability(para_id: ParaId) -> Option> { - parachains_runtime_api_impl::candidate_pending_availability::(para_id) - } - - fn candidate_events() -> Vec> { - parachains_runtime_api_impl::candidate_events::(|ev| { - match ev { - RuntimeEvent::ParaInclusion(ev) => { - Some(ev) - } - _ => None, - } - }) - } - - fn session_info(index: SessionIndex) -> Option { - parachains_runtime_api_impl::session_info::(index) - } - - fn session_executor_params(session_index: SessionIndex) -> Option { - parachains_runtime_api_impl::session_executor_params::(session_index) - } - - fn dmq_contents(recipient: ParaId) -> Vec> { - parachains_runtime_api_impl::dmq_contents::(recipient) - } - - fn inbound_hrmp_channels_contents( - recipient: ParaId - ) -> BTreeMap>> { - parachains_runtime_api_impl::inbound_hrmp_channels_contents::(recipient) - } - - fn validation_code_by_hash(hash: ValidationCodeHash) -> Option { - parachains_runtime_api_impl::validation_code_by_hash::(hash) - } - - fn on_chain_votes() -> Option> { - parachains_runtime_api_impl::on_chain_votes::() - } - - fn submit_pvf_check_statement( - stmt: primitives::PvfCheckStatement, - signature: primitives::ValidatorSignature, - ) { - parachains_runtime_api_impl::submit_pvf_check_statement::(stmt, signature) - } - - fn pvfs_require_precheck() -> Vec { - parachains_runtime_api_impl::pvfs_require_precheck::() - } - - fn validation_code_hash(para_id: ParaId, assumption: OccupiedCoreAssumption) - -> Option - { - parachains_runtime_api_impl::validation_code_hash::(para_id, assumption) - } - - fn disputes() -> Vec<(SessionIndex, CandidateHash, DisputeState)> { - parachains_runtime_api_impl::get_session_disputes::() - } - - fn unapplied_slashes( - ) -> Vec<(SessionIndex, CandidateHash, slashing::PendingSlashes)> { - parachains_runtime_api_impl::unapplied_slashes::() - } - - fn key_ownership_proof( - validator_id: ValidatorId, - ) -> Option { - use parity_scale_codec::Encode; - - Historical::prove((PARACHAIN_KEY_TYPE_ID, validator_id)) - .map(|p| p.encode()) - .map(slashing::OpaqueKeyOwnershipProof::new) - } - - fn submit_report_dispute_lost( - dispute_proof: slashing::DisputeProof, - key_ownership_proof: slashing::OpaqueKeyOwnershipProof, - ) -> Option<()> { - parachains_runtime_api_impl::submit_unsigned_slashing_report::( - dispute_proof, - key_ownership_proof, - ) - } - } - - impl beefy_primitives::BeefyApi for Runtime { - fn beefy_genesis() -> Option { - // dummy implementation due to lack of BEEFY pallet. - None - } - - fn validator_set() -> Option> { - // dummy implementation due to lack of BEEFY pallet. - None - } - - fn submit_report_equivocation_unsigned_extrinsic( - _equivocation_proof: beefy_primitives::EquivocationProof< - BlockNumber, - BeefyId, - BeefySignature, - >, - _key_owner_proof: beefy_primitives::OpaqueKeyOwnershipProof, - ) -> Option<()> { - None - } - - fn generate_key_ownership_proof( - _set_id: beefy_primitives::ValidatorSetId, - _authority_id: BeefyId, - ) -> Option { - None - } - } - - impl mmr::MmrApi for Runtime { - fn mmr_root() -> Result { - Err(mmr::Error::PalletNotIncluded) - } - - fn mmr_leaf_count() -> Result { - Err(mmr::Error::PalletNotIncluded) - } - - fn generate_proof( - _block_numbers: Vec, - _best_known_block_number: Option, - ) -> Result<(Vec, mmr::Proof), mmr::Error> { - Err(mmr::Error::PalletNotIncluded) - } - - fn verify_proof(_leaves: Vec, _proof: mmr::Proof) - -> Result<(), mmr::Error> - { - Err(mmr::Error::PalletNotIncluded) - } - - fn verify_proof_stateless( - _root: Hash, - _leaves: Vec, - _proof: mmr::Proof - ) -> Result<(), mmr::Error> { - Err(mmr::Error::PalletNotIncluded) - } - } - - impl fg_primitives::GrandpaApi for Runtime { - fn grandpa_authorities() -> Vec<(GrandpaId, u64)> { - Grandpa::grandpa_authorities() - } - - fn current_set_id() -> fg_primitives::SetId { - Grandpa::current_set_id() - } - - fn submit_report_equivocation_unsigned_extrinsic( - equivocation_proof: fg_primitives::EquivocationProof< - ::Hash, - sp_runtime::traits::NumberFor, - >, - key_owner_proof: fg_primitives::OpaqueKeyOwnershipProof, - ) -> Option<()> { - let key_owner_proof = key_owner_proof.decode()?; - - Grandpa::submit_unsigned_equivocation_report( - equivocation_proof, - key_owner_proof, - ) - } - - fn generate_key_ownership_proof( - _set_id: fg_primitives::SetId, - authority_id: fg_primitives::AuthorityId, - ) -> Option { - use parity_scale_codec::Encode; - - Historical::prove((fg_primitives::KEY_TYPE, authority_id)) - .map(|p| p.encode()) - .map(fg_primitives::OpaqueKeyOwnershipProof::new) - } - } - - impl babe_primitives::BabeApi for Runtime { - fn configuration() -> babe_primitives::BabeConfiguration { - let epoch_config = Babe::epoch_config().unwrap_or(BABE_GENESIS_EPOCH_CONFIG); - babe_primitives::BabeConfiguration { - slot_duration: Babe::slot_duration(), - epoch_length: EpochDuration::get(), - c: epoch_config.c, - authorities: Babe::authorities().to_vec(), - randomness: Babe::randomness(), - allowed_slots: epoch_config.allowed_slots, - } - } - - fn current_epoch_start() -> babe_primitives::Slot { - Babe::current_epoch_start() - } - - fn current_epoch() -> babe_primitives::Epoch { - Babe::current_epoch() - } - - fn next_epoch() -> babe_primitives::Epoch { - Babe::next_epoch() - } - - fn generate_key_ownership_proof( - _slot: babe_primitives::Slot, - authority_id: babe_primitives::AuthorityId, - ) -> Option { - use parity_scale_codec::Encode; - - Historical::prove((babe_primitives::KEY_TYPE, authority_id)) - .map(|p| p.encode()) - .map(babe_primitives::OpaqueKeyOwnershipProof::new) - } - - fn submit_report_equivocation_unsigned_extrinsic( - equivocation_proof: babe_primitives::EquivocationProof<::Header>, - key_owner_proof: babe_primitives::OpaqueKeyOwnershipProof, - ) -> Option<()> { - let key_owner_proof = key_owner_proof.decode()?; - - Babe::submit_unsigned_equivocation_report( - equivocation_proof, - key_owner_proof, - ) - } - } - - impl authority_discovery_primitives::AuthorityDiscoveryApi for Runtime { - fn authorities() -> Vec { - parachains_runtime_api_impl::relevant_authority_ids::() - } - } - - impl sp_session::SessionKeys for Runtime { - fn generate_session_keys(seed: Option>) -> Vec { - SessionKeys::generate(seed) - } - - fn decode_session_keys( - encoded: Vec, - ) -> Option, sp_core::crypto::KeyTypeId)>> { - SessionKeys::decode_into_raw_public_keys(&encoded) - } - } - - impl frame_system_rpc_runtime_api::AccountNonceApi for Runtime { - fn account_nonce(account: AccountId) -> Nonce { - System::account_nonce(account) - } - } - - impl pallet_transaction_payment_rpc_runtime_api::TransactionPaymentApi< - Block, - Balance, - > for Runtime { - fn query_info(uxt: ::Extrinsic, len: u32) -> RuntimeDispatchInfo { - TransactionPayment::query_info(uxt, len) - } - fn query_fee_details(uxt: ::Extrinsic, len: u32) -> FeeDetails { - TransactionPayment::query_fee_details(uxt, len) - } - fn query_weight_to_fee(weight: Weight) -> Balance { - TransactionPayment::weight_to_fee(weight) - } - fn query_length_to_fee(length: u32) -> Balance { - TransactionPayment::length_to_fee(length) - } - } - - impl pallet_transaction_payment_rpc_runtime_api::TransactionPaymentCallApi - for Runtime - { - fn query_call_info(call: RuntimeCall, len: u32) -> RuntimeDispatchInfo { - TransactionPayment::query_call_info(call, len) - } - fn query_call_fee_details(call: RuntimeCall, len: u32) -> FeeDetails { - TransactionPayment::query_call_fee_details(call, len) - } - fn query_weight_to_fee(weight: Weight) -> Balance { - TransactionPayment::weight_to_fee(weight) - } - fn query_length_to_fee(length: u32) -> Balance { - TransactionPayment::length_to_fee(length) - } - } - - #[cfg(feature = "try-runtime")] - impl frame_try_runtime::TryRuntime for Runtime { - fn on_runtime_upgrade(checks: frame_try_runtime::UpgradeCheckSelect) -> (Weight, Weight) { - log::info!("try-runtime::on_runtime_upgrade polkadot."); - let weight = Executive::try_runtime_upgrade(checks).unwrap(); - (weight, BlockWeights::get().max_block) - } - - fn execute_block( - block: Block, - state_root_check: bool, - signature_check: bool, - select: frame_try_runtime::TryStateSelect, - ) -> Weight { - // NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to - // have a backtrace here. - Executive::try_execute_block(block, state_root_check, signature_check, select).unwrap() - } - } - - #[cfg(feature = "runtime-benchmarks")] - impl frame_benchmarking::Benchmark for Runtime { - fn benchmark_metadata(extra: bool) -> ( - Vec, - Vec, - ) { - use frame_benchmarking::{Benchmarking, BenchmarkList}; - use frame_support::traits::StorageInfoTrait; - - use pallet_session_benchmarking::Pallet as SessionBench; - use pallet_offences_benchmarking::Pallet as OffencesBench; - use pallet_election_provider_support_benchmarking::Pallet as ElectionProviderBench; - use pallet_nomination_pools_benchmarking::Pallet as NominationPoolsBench; - use frame_system_benchmarking::Pallet as SystemBench; - use frame_benchmarking::baseline::Pallet as Baseline; - - let mut list = Vec::::new(); - list_benchmarks!(list, extra); - - let storage_info = AllPalletsWithSystem::storage_info(); - return (list, storage_info) - } - - fn dispatch_benchmark( - config: frame_benchmarking::BenchmarkConfig - ) -> Result< - Vec, - sp_runtime::RuntimeString, - > { - use frame_support::traits::WhitelistedStorageKeys; - use frame_benchmarking::{Benchmarking, BenchmarkBatch, BenchmarkError}; - use sp_storage::TrackedStorageKey; - // Trying to add benchmarks directly to some pallets caused cyclic dependency issues. - // To get around that, we separated the benchmarks into its own crate. - use pallet_session_benchmarking::Pallet as SessionBench; - use pallet_offences_benchmarking::Pallet as OffencesBench; - use pallet_election_provider_support_benchmarking::Pallet as ElectionProviderBench; - use pallet_nomination_pools_benchmarking::Pallet as NominationPoolsBench; - use frame_system_benchmarking::Pallet as SystemBench; - use frame_benchmarking::baseline::Pallet as Baseline; - use xcm::latest::prelude::*; - use xcm_config::{XcmConfig, StatemintLocation, TokenLocation, LocalCheckAccount, SovereignAccountOf}; - - impl pallet_session_benchmarking::Config for Runtime {} - impl pallet_offences_benchmarking::Config for Runtime {} - impl pallet_election_provider_support_benchmarking::Config for Runtime {} - impl frame_system_benchmarking::Config for Runtime {} - impl frame_benchmarking::baseline::Config for Runtime {} - impl pallet_nomination_pools_benchmarking::Config for Runtime {} - impl runtime_parachains::disputes::slashing::benchmarking::Config for Runtime {} - - let mut whitelist: Vec = AllPalletsWithSystem::whitelisted_storage_keys(); - let treasury_key = frame_system::Account::::hashed_key_for(Treasury::account_id()); - whitelist.push(treasury_key.to_vec().into()); - - impl pallet_xcm_benchmarks::Config for Runtime { - type XcmConfig = XcmConfig; - type AccountIdConverter = SovereignAccountOf; - fn valid_destination() -> Result { - Ok(StatemintLocation::get()) - } - fn worst_case_holding(_depositable_count: u32) -> MultiAssets { - // Polkadot only knows about DOT - vec![MultiAsset { id: Concrete(TokenLocation::get()), fun: Fungible(1_000_000 * UNITS) }].into() - } - } - - parameter_types! { - pub const TrustedTeleporter: Option<(MultiLocation, MultiAsset)> = Some(( - StatemintLocation::get(), - MultiAsset { id: Concrete(TokenLocation::get()), fun: Fungible(1 * UNITS) } - )); - pub const TrustedReserve: Option<(MultiLocation, MultiAsset)> = None; - } - - impl pallet_xcm_benchmarks::fungible::Config for Runtime { - type TransactAsset = Balances; - - type CheckedAccount = LocalCheckAccount; - type TrustedTeleporter = TrustedTeleporter; - type TrustedReserve = TrustedReserve; - - fn get_multi_asset() -> MultiAsset { - MultiAsset { - id: Concrete(TokenLocation::get()), - fun: Fungible(1 * UNITS) - } - } - } - - impl pallet_xcm_benchmarks::generic::Config for Runtime { - type RuntimeCall = RuntimeCall; - - fn worst_case_response() -> (u64, Response) { - (0u64, Response::Version(Default::default())) - } - - fn worst_case_asset_exchange() -> Result<(MultiAssets, MultiAssets), BenchmarkError> { - // Polkadot doesn't support asset exchanges - Err(BenchmarkError::Skip) - } - - fn universal_alias() -> Result<(MultiLocation, Junction), BenchmarkError> { - // The XCM executor of Polkadot doesn't have a configured `UniversalAliases` - Err(BenchmarkError::Skip) - } - - fn transact_origin_and_runtime_call() -> Result<(MultiLocation, RuntimeCall), BenchmarkError> { - Ok((StatemintLocation::get(), frame_system::Call::remark_with_event { remark: vec![] }.into())) - } - - fn subscribe_origin() -> Result { - Ok(StatemintLocation::get()) - } - - fn claimable_asset() -> Result<(MultiLocation, MultiLocation, MultiAssets), BenchmarkError> { - let origin = StatemintLocation::get(); - let assets: MultiAssets = (Concrete(TokenLocation::get()), 1_000 * UNITS).into(); - let ticket = MultiLocation { parents: 0, interior: Here }; - Ok((origin, ticket, assets)) - } - - fn unlockable_asset() -> Result<(MultiLocation, MultiLocation, MultiAsset), BenchmarkError> { - // Polkadot doesn't support asset locking - Err(BenchmarkError::Skip) - } - - fn export_message_origin_and_destination( - ) -> Result<(MultiLocation, NetworkId, InteriorMultiLocation), BenchmarkError> { - // Polkadot doesn't support exporting messages - Err(BenchmarkError::Skip) - } - - fn alias_origin() -> Result<(MultiLocation, MultiLocation), BenchmarkError> { - // The XCM executor of Polkadot doesn't have a configured `Aliasers` - Err(BenchmarkError::Skip) - } - } - - let mut batches = Vec::::new(); - let params = (&config, &whitelist); - - add_benchmarks!(params, batches); - - Ok(batches) - } - } -} - -#[cfg(test)] -mod test_fees { - use super::*; - use frame_support::{dispatch::GetDispatchInfo, weights::WeightToFee as WeightToFeeT}; - use keyring::Sr25519Keyring::{Alice, Charlie}; - use pallet_transaction_payment::Multiplier; - use runtime_common::MinimumMultiplier; - use separator::Separatable; - use sp_runtime::{assert_eq_error_rate, FixedPointNumber, MultiAddress, MultiSignature}; - - #[test] - fn payout_weight_portion() { - use pallet_staking::WeightInfo; - let payout_weight = - ::WeightInfo::payout_stakers_alive_staked( - MaxNominatorRewardedPerValidator::get(), - ) - .ref_time() as f64; - let block_weight = BlockWeights::get().max_block.ref_time() as f64; - - println!( - "a full payout takes {:.2} of the block weight [{} / {}]", - payout_weight / block_weight, - payout_weight, - block_weight - ); - assert!(payout_weight * 2f64 < block_weight); - } - - #[test] - fn block_cost() { - let max_block_weight = BlockWeights::get().max_block; - let raw_fee = WeightToFee::weight_to_fee(&max_block_weight); - - let fee_with_multiplier = |m: Multiplier| { - println!( - "Full Block weight == {} // multiplier: {:?} // WeightToFee(full_block) == {} plank", - max_block_weight, - m, - m.saturating_mul_int(raw_fee).separated_string(), - ); - }; - fee_with_multiplier(MinimumMultiplier::get()); - fee_with_multiplier(Multiplier::from_rational(1, 2)); - fee_with_multiplier(Multiplier::from_u32(1)); - fee_with_multiplier(Multiplier::from_u32(2)); - } - - #[test] - fn transfer_cost_min_multiplier() { - let min_multiplier = MinimumMultiplier::get(); - let call = pallet_balances::Call::::transfer_keep_alive { - dest: Charlie.to_account_id().into(), - value: Default::default(), - }; - let info = call.get_dispatch_info(); - println!("call = {:?} / info = {:?}", call, info); - // convert to runtime call. - let call = RuntimeCall::Balances(call); - let extra: SignedExtra = ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(generic::Era::immortal()), - frame_system::CheckNonce::::from(1), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(0), - claims::PrevalidateAttests::::new(), - ); - let uxt = UncheckedExtrinsic { - function: call, - signature: Some(( - MultiAddress::Id(Alice.to_account_id()), - MultiSignature::Sr25519(Alice.sign(b"foo")), - extra, - )), - }; - let len = uxt.encoded_size(); - - let mut ext = sp_io::TestExternalities::new_empty(); - let mut test_with_multiplier = |m: Multiplier| { - ext.execute_with(|| { - pallet_transaction_payment::NextFeeMultiplier::::put(m); - let fee = TransactionPayment::query_fee_details(uxt.clone(), len as u32); - println!( - "multiplier = {:?} // fee details = {:?} // final fee = {:?}", - pallet_transaction_payment::NextFeeMultiplier::::get(), - fee, - fee.final_fee().separated_string(), - ); - }); - }; - - test_with_multiplier(min_multiplier); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_0u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_00u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_000u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_000_000u128)); - test_with_multiplier(Multiplier::saturating_from_rational(1u128, 1_000_000_000u128)); - } - - #[test] - fn nominator_limit() { - use pallet_election_provider_multi_phase::WeightInfo; - // starting point of the nominators. - let target_voters: u32 = 50_000; - - // assuming we want around 5k candidates and 1k active validators. (March 31, 2021) - let all_targets: u32 = 5_000; - let desired: u32 = 1_000; - let weight_with = |active| { - ::WeightInfo::submit_unsigned( - active, - all_targets, - active, - desired, - ) - }; - - let mut active = target_voters; - while weight_with(active).all_lte(OffchainSolutionWeightLimit::get()) || - active == target_voters - { - active += 1; - } - - println!("can support {} nominators to yield a weight of {}", active, weight_with(active)); - assert!(active > target_voters, "we need to reevaluate the weight of the election system"); - } - - #[test] - fn signed_deposit_is_sensible() { - // ensure this number does not change, or that it is checked after each change. - // a 1 MB solution should take (40 + 10) DOTs of deposit. - let deposit = SignedFixedDeposit::get() + (SignedDepositByte::get() * 1024 * 1024); - assert_eq_error_rate!(deposit, 50 * DOLLARS, DOLLARS); - } -} - -#[cfg(test)] -mod test { - use std::collections::HashSet; - - use super::*; - use frame_support::traits::WhitelistedStorageKeys; - use sp_core::hexdisplay::HexDisplay; - - #[test] - fn call_size() { - RuntimeCall::assert_size_under(230); - } - - #[test] - fn check_whitelist() { - let whitelist: HashSet = AllPalletsWithSystem::whitelisted_storage_keys() - .iter() - .map(|e| HexDisplay::from(&e.key).to_string()) - .collect(); - - // Block number - assert!( - whitelist.contains("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac") - ); - // Total issuance - assert!( - whitelist.contains("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80") - ); - // Execution phase - assert!( - whitelist.contains("26aa394eea5630e07c48ae0c9558cef7ff553b5a9862a516939d82b3d3d8661a") - ); - // Event count - assert!( - whitelist.contains("26aa394eea5630e07c48ae0c9558cef70a98fdbe9ce6c55837576c60c7af3850") - ); - // System events - assert!( - whitelist.contains("26aa394eea5630e07c48ae0c9558cef780d41e5e16056765bc8461851072c9d7") - ); - // XcmPallet VersionDiscoveryQueue - assert!( - whitelist.contains("1405f2411d0af5a7ff397e7c9dc68d194a222ba0333561192e474c59ed8e30e1") - ); - // XcmPallet SafeXcmVersion - assert!( - whitelist.contains("1405f2411d0af5a7ff397e7c9dc68d196323ae84c43568be0d1394d5d0d522c4") - ); - } -} - -#[cfg(test)] -mod multiplier_tests { - use super::*; - use frame_support::{dispatch::DispatchInfo, traits::OnFinalize}; - use runtime_common::{MinimumMultiplier, TargetBlockFullness}; - use separator::Separatable; - use sp_runtime::traits::Convert; - - fn run_with_system_weight(w: Weight, mut assertions: F) - where - F: FnMut() -> (), - { - let mut t: sp_io::TestExternalities = frame_system::GenesisConfig::::default() - .build_storage() - .unwrap() - .into(); - t.execute_with(|| { - System::set_block_consumed_resources(w, 0); - assertions() - }); - } - - #[test] - fn multiplier_can_grow_from_zero() { - let minimum_multiplier = MinimumMultiplier::get(); - let target = TargetBlockFullness::get() * - BlockWeights::get().get(DispatchClass::Normal).max_total.unwrap(); - // if the min is too small, then this will not change, and we are doomed forever. - // the weight is 1/100th bigger than target. - run_with_system_weight(target.saturating_mul(101) / 100, || { - let next = SlowAdjustingFeeUpdate::::convert(minimum_multiplier); - assert!(next > minimum_multiplier, "{:?} !>= {:?}", next, minimum_multiplier); - }) - } - - #[test] - fn fast_unstake_estimate() { - use pallet_fast_unstake::WeightInfo; - let block_time = BlockWeights::get().max_block.ref_time() as f32; - let on_idle = weights::pallet_fast_unstake::WeightInfo::::on_idle_check( - 300, - ::BatchSize::get(), - ) - .ref_time() as f32; - println!("ratio of block weight for full batch fast-unstake {}", on_idle / block_time); - assert!(on_idle / block_time <= 0.5f32) - } - - #[test] - #[ignore] - fn multiplier_growth_simulator() { - // assume the multiplier is initially set to its minimum. We update it with values twice the - //target (target is 25%, thus 50%) and we see at which point it reaches 1. - let mut multiplier = MinimumMultiplier::get(); - let block_weight = BlockWeights::get().get(DispatchClass::Normal).max_total.unwrap(); - let mut blocks = 0; - let mut fees_paid = 0; - - frame_system::Pallet::::set_block_consumed_resources(Weight::MAX, 0); - let info = DispatchInfo { weight: Weight::MAX, ..Default::default() }; - - let mut t: sp_io::TestExternalities = frame_system::GenesisConfig::::default() - .build_storage() - .unwrap() - .into(); - // set the minimum - t.execute_with(|| { - pallet_transaction_payment::NextFeeMultiplier::::set(MinimumMultiplier::get()); - }); - - while multiplier <= Multiplier::from_u32(1) { - t.execute_with(|| { - // imagine this tx was called. - let fee = TransactionPayment::compute_fee(0, &info, 0); - fees_paid += fee; - - // this will update the multiplier. - System::set_block_consumed_resources(block_weight, 0); - TransactionPayment::on_finalize(1); - let next = TransactionPayment::next_fee_multiplier(); - - assert!(next > multiplier, "{:?} !>= {:?}", next, multiplier); - multiplier = next; - - println!( - "block = {} / multiplier {:?} / fee = {:?} / fess so far {:?}", - blocks, - multiplier, - fee.separated_string(), - fees_paid.separated_string() - ); - }); - blocks += 1; - } - } - - #[test] - #[ignore] - fn multiplier_cool_down_simulator() { - // assume the multiplier is initially set to its minimum. We update it with values twice the - //target (target is 25%, thus 50%) and we see at which point it reaches 1. - let mut multiplier = Multiplier::from_u32(2); - let mut blocks = 0; - - let mut t: sp_io::TestExternalities = frame_system::GenesisConfig::::default() - .build_storage() - .unwrap() - .into(); - // set the minimum - t.execute_with(|| { - pallet_transaction_payment::NextFeeMultiplier::::set(multiplier); - }); - - while multiplier > Multiplier::from_u32(0) { - t.execute_with(|| { - // this will update the multiplier. - TransactionPayment::on_finalize(1); - let next = TransactionPayment::next_fee_multiplier(); - - assert!(next < multiplier, "{:?} !>= {:?}", next, multiplier); - multiplier = next; - - println!("block = {} / multiplier {:?}", blocks, multiplier); - }); - blocks += 1; - } - } -} - -#[cfg(all(test, feature = "try-runtime"))] -mod remote_tests { - use super::*; - use frame_try_runtime::{runtime_decl_for_try_runtime::TryRuntime, UpgradeCheckSelect}; - use remote_externalities::{ - Builder, Mode, OfflineConfig, OnlineConfig, SnapshotConfig, Transport, - }; - use std::env::var; - - #[tokio::test] - async fn run_migrations() { - if var("RUN_MIGRATION_TESTS").is_err() { - return - } - - sp_tracing::try_init_simple(); - let transport: Transport = - var("WS").unwrap_or("wss://rpc.polkadot.io:443".to_string()).into(); - let maybe_state_snapshot: Option = var("SNAP").map(|s| s.into()).ok(); - let mut ext = Builder::::default() - .mode(if let Some(state_snapshot) = maybe_state_snapshot { - Mode::OfflineOrElseOnline( - OfflineConfig { state_snapshot: state_snapshot.clone() }, - OnlineConfig { - transport, - state_snapshot: Some(state_snapshot), - ..Default::default() - }, - ) - } else { - Mode::Online(OnlineConfig { transport, ..Default::default() }) - }) - .build() - .await - .unwrap(); - ext.execute_with(|| Runtime::on_runtime_upgrade(UpgradeCheckSelect::PreAndPost)); - } - - #[tokio::test] - #[ignore = "this test is meant to be executed manually"] - async fn try_fast_unstake_all() { - sp_tracing::try_init_simple(); - let transport: Transport = - var("WS").unwrap_or("wss://rpc.polkadot.io:443".to_string()).into(); - let maybe_state_snapshot: Option = var("SNAP").map(|s| s.into()).ok(); - let mut ext = Builder::::default() - .mode(if let Some(state_snapshot) = maybe_state_snapshot { - Mode::OfflineOrElseOnline( - OfflineConfig { state_snapshot: state_snapshot.clone() }, - OnlineConfig { - transport, - state_snapshot: Some(state_snapshot), - ..Default::default() - }, - ) - } else { - Mode::Online(OnlineConfig { transport, ..Default::default() }) - }) - .build() - .await - .unwrap(); - ext.execute_with(|| { - pallet_fast_unstake::ErasToCheckPerBlock::::put(1); - runtime_common::try_runtime::migrate_all_inactive_nominators::() - }); - } -} From fc286188b61dfa351cf41b48318b77eba3512878 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 15:21:38 +0300 Subject: [PATCH 44/89] Remove the need for CollationVersion::VStaging Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/bridge/src/network.rs | 20 --------- polkadot/node/network/bridge/src/rx/mod.rs | 33 +------------- polkadot/node/network/bridge/src/tx/mod.rs | 20 ++------- .../src/collator_side/mod.rs | 45 +++---------------- .../src/validator_side/mod.rs | 27 +++-------- polkadot/node/network/protocol/src/lib.rs | 18 +++----- .../node/network/protocol/src/peer_set.rs | 4 -- 7 files changed, 26 insertions(+), 141 deletions(-) diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index a13045285522..c264c94cc19b 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -145,26 +145,6 @@ pub(crate) fn send_collation_message_v2( ); } -// Helper function to send a collation vstaging message to a list of peers. -// Messages are always sent via the main protocol, even legacy protocol messages. -pub(crate) fn send_collation_message_vstaging( - net: &mut impl Network, - peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, - message: WireMessage, - metrics: &Metrics, -) { - send_message( - net, - peers, - PeerSet::Collation, - CollationVersion::VStaging.into(), - peerset_protocol_names, - message, - metrics, - ); -} - /// Lower level function that sends a message to the network using the main protocol version. /// /// This function is only used internally by the network-bridge, which is responsible to only send diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 83bf10bbb670..acb664a8ab29 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -66,9 +66,8 @@ use super::validator_discovery; /// /// Defines the `Network` trait with an implementation for an `Arc`. use crate::network::{ - send_collation_message_v1, send_collation_message_v2, send_collation_message_vstaging, - send_validation_message_v1, send_validation_message_v2, send_validation_message_vstaging, - Network, + send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, + send_validation_message_v2, send_validation_message_vstaging, Network, }; use crate::{network::get_peer_id_by_authority_id, WireMessage}; @@ -320,15 +319,6 @@ where ), &metrics, ), - CollationVersion::VStaging => send_collation_message_vstaging( - &mut network_service, - vec![peer], - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), } }, } @@ -548,16 +538,6 @@ where c_messages, &metrics, ) - } else if expected_versions[PeerSet::Collation] == - Some(CollationVersion::VStaging.into()) - { - handle_peer_messages::( - remote, - PeerSet::Collation, - &mut shared.0.lock().collation_peers, - c_messages, - &metrics, - ) } else { gum::warn!( target: LOG_TARGET, @@ -858,8 +838,6 @@ fn update_our_view( let vstaging_validation_peers = filter_by_peer_version(&validation_peers, ValidationVersion::VStaging.into()); - let vstaging_collation_peers = - filter_by_peer_version(&collation_peers, CollationVersion::VStaging.into()); send_validation_message_v1( net, @@ -901,13 +879,6 @@ fn update_our_view( metrics, ); - send_collation_message_vstaging( - net, - vstaging_collation_peers, - peerset_protocol_names, - WireMessage::ViewUpdate(new_view.clone()), - metrics, - ); let our_view = OurView::new( live_heads.iter().take(MAX_VIEW_HEADS).cloned().map(|a| (a.hash, a.span)), finalized_number, diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index 603869a4e868..5f222ad59c75 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -34,7 +34,7 @@ pub use polkadot_node_network_protocol::peer_set::{peer_sets_info, IsAuthority}; use polkadot_node_network_protocol::request_response::Requests; use sc_network::ReputationChange; -use crate::{network::send_collation_message_vstaging, validator_discovery}; +use crate::validator_discovery; /// Actual interfacing to the network based on the `Network` trait. /// @@ -265,14 +265,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::V2(msg) => send_collation_message_v2( - &mut network_service, - peers, - peerset_protocol_names, - WireMessage::ProtocolMessage(msg), - &metrics, - ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, @@ -297,14 +290,7 @@ where WireMessage::ProtocolMessage(msg), &metrics, ), - Versioned::V2(msg) => send_collation_message_v2( - &mut network_service, - peers, - peerset_protocol_names, - WireMessage::ProtocolMessage(msg), - &metrics, - ), - Versioned::VStaging(msg) => send_collation_message_vstaging( + Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( &mut network_service, peers, peerset_protocol_names, diff --git a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs index 1c40759dde7f..4fb24ef30bab 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs @@ -33,8 +33,8 @@ use polkadot_node_network_protocol::{ incoming::{self, OutgoingResponse}, v1 as request_v1, v2 as request_v2, IncomingRequestReceiver, }, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, - UnifiedReputationChange as Rep, Versioned, View, + v1 as protocol_v1, v2 as protocol_v2, OurView, PeerId, UnifiedReputationChange as Rep, + Versioned, View, }; use polkadot_node_primitives::{CollationSecondedSignal, PoV, Statement}; use polkadot_node_subsystem::{ @@ -577,13 +577,7 @@ async fn determine_our_validators( fn declare_message( state: &mut State, version: CollationVersion, -) -> Option< - Versioned< - protocol_v1::CollationProtocol, - protocol_v2::CollationProtocol, - protocol_vstaging::CollationProtocol, - >, -> { +) -> Option> { let para_id = state.collating_on?; Some(match version { CollationVersion::V1 => { @@ -606,16 +600,6 @@ fn declare_message( ); Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) }, - CollationVersion::VStaging => { - let declare_signature_payload = - protocol_vstaging::declare_signature_payload(&state.local_peer_id); - let wire_message = protocol_vstaging::CollatorProtocolMessage::Declare( - state.collator_pair.public(), - para_id, - state.collator_pair.sign(&declare_signature_payload), - ); - Versioned::VStaging(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) - }, }) } @@ -728,16 +712,6 @@ async fn advertise_collation( }; Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol(wire_message)) }, - CollationVersion::VStaging => { - let wire_message = protocol_vstaging::CollatorProtocolMessage::AdvertiseCollation { - relay_parent, - candidate_hash: *candidate_hash, - parent_head_data_hash: collation.parent_head_data_hash, - }; - Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( - wire_message, - )) - }, CollationVersion::V1 => { let wire_message = protocol_v1::CollatorProtocolMessage::AdvertiseCollation(relay_parent); @@ -890,20 +864,15 @@ async fn handle_incoming_peer_message( runtime: &mut RuntimeInfo, state: &mut State, origin: PeerId, - msg: Versioned< - protocol_v1::CollatorProtocolMessage, - protocol_v2::CollatorProtocolMessage, - protocol_vstaging::CollatorProtocolMessage, - >, + msg: Versioned, ) -> Result<()> { use protocol_v1::CollatorProtocolMessage as V1; use protocol_v2::CollatorProtocolMessage as V2; - use protocol_vstaging::CollatorProtocolMessage as VStaging; match msg { Versioned::V1(V1::Declare(..)) | Versioned::V2(V2::Declare(..)) | - Versioned::VStaging(VStaging::Declare(..)) => { + Versioned::VStaging(V2::Declare(..)) => { gum::trace!( target: LOG_TARGET, ?origin, @@ -916,7 +885,7 @@ async fn handle_incoming_peer_message( }, Versioned::V1(V1::AdvertiseCollation(_)) | Versioned::V2(V2::AdvertiseCollation { .. }) | - Versioned::VStaging(VStaging::AdvertiseCollation { .. }) => { + Versioned::VStaging(V2::AdvertiseCollation { .. }) => { gum::trace!( target: LOG_TARGET, ?origin, @@ -932,7 +901,7 @@ async fn handle_incoming_peer_message( }, Versioned::V1(V1::CollationSeconded(relay_parent, statement)) | Versioned::V2(V2::CollationSeconded(relay_parent, statement)) | - Versioned::VStaging(VStaging::CollationSeconded(relay_parent, statement)) => { + Versioned::VStaging(V2::CollationSeconded(relay_parent, statement)) => { if !matches!(statement.unchecked_payload(), Statement::Seconded(_)) { gum::warn!( target: LOG_TARGET, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 0567b33dfe85..56b2829b5e61 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -36,8 +36,8 @@ use polkadot_node_network_protocol::{ outgoing::{Recipient, RequestError}, v1 as request_v1, v2 as request_v2, OutgoingRequest, Requests, }, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, - UnifiedReputationChange as Rep, Versioned, View, + v1 as protocol_v1, v2 as protocol_v2, OurView, PeerId, UnifiedReputationChange as Rep, + Versioned, View, }; use polkadot_node_primitives::{SignedFullStatement, Statement}; use polkadot_node_subsystem::{ @@ -627,13 +627,6 @@ async fn notify_collation_seconded( CollationVersion::V2 => Versioned::V2(protocol_v2::CollationProtocol::CollatorProtocol( protocol_v2::CollatorProtocolMessage::CollationSeconded(relay_parent, statement), )), - CollationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::CollationProtocol::CollatorProtocol( - protocol_vstaging::CollatorProtocolMessage::CollationSeconded( - relay_parent, - statement, - ), - )), }; sender .send_message(NetworkBridgeTxMessage::SendCollationMessage(vec![peer_id], wire_message)) @@ -697,8 +690,7 @@ async fn request_collation( let requests = Requests::CollationFetchingV1(req); (requests, response_recv.boxed()) }, - (CollationVersion::V2, Some(ProspectiveCandidate { candidate_hash, .. })) | - (CollationVersion::VStaging, Some(ProspectiveCandidate { candidate_hash, .. })) => { + (CollationVersion::V2, Some(ProspectiveCandidate { candidate_hash, .. })) => { let (req, response_recv) = OutgoingRequest::new( Recipient::Peer(peer_id), request_v2::CollationFetchingRequest { relay_parent, para_id, candidate_hash }, @@ -758,21 +750,16 @@ async fn process_incoming_peer_message( ctx: &mut Context, state: &mut State, origin: PeerId, - msg: Versioned< - protocol_v1::CollatorProtocolMessage, - protocol_v2::CollatorProtocolMessage, - protocol_vstaging::CollatorProtocolMessage, - >, + msg: Versioned, ) { use protocol_v1::CollatorProtocolMessage as V1; use protocol_v2::CollatorProtocolMessage as V2; - use protocol_vstaging::CollatorProtocolMessage as VStaging; use sp_runtime::traits::AppVerify; match msg { Versioned::V1(V1::Declare(collator_id, para_id, signature)) | Versioned::V2(V2::Declare(collator_id, para_id, signature)) | - Versioned::VStaging(VStaging::Declare(collator_id, para_id, signature)) => { + Versioned::VStaging(V2::Declare(collator_id, para_id, signature)) => { if collator_peer_id(&state.peer_data, &collator_id).is_some() { modify_reputation( &mut state.reputation, @@ -889,7 +876,7 @@ async fn process_incoming_peer_message( candidate_hash, parent_head_data_hash, }) | - Versioned::VStaging(VStaging::AdvertiseCollation { + Versioned::VStaging(V2::AdvertiseCollation { relay_parent, candidate_hash, parent_head_data_hash, @@ -918,7 +905,7 @@ async fn process_incoming_peer_message( }, Versioned::V1(V1::CollationSeconded(..)) | Versioned::V2(V2::CollationSeconded(..)) | - Versioned::VStaging(VStaging::CollationSeconded(..)) => { + Versioned::VStaging(V2::CollationSeconded(..)) => { gum::warn!( target: LOG_TARGET, peer_id = ?origin, diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index f01de9862ce8..a7251f7bf8ed 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -253,7 +253,7 @@ impl View { /// A protocol-versioned type. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Versioned { +pub enum Versioned { /// V1 type. V1(V1), /// V2 type. @@ -296,8 +296,7 @@ impl From for VersionedValidationProtocol { } /// All supported versions of the collation protocol message. -pub type VersionedCollationProtocol = - Versioned; +pub type VersionedCollationProtocol = Versioned; impl From for VersionedCollationProtocol { fn from(v1: v1::CollationProtocol) -> Self { @@ -446,11 +445,8 @@ impl<'a> TryFrom<&'a VersionedValidationProtocol> for GossipSupportNetworkMessag } /// Version-annotated messages used by the bitfield distribution subsystem. -pub type CollatorProtocolMessage = Versioned< - v1::CollatorProtocolMessage, - v2::CollatorProtocolMessage, - vstaging::CollatorProtocolMessage, ->; +pub type CollatorProtocolMessage = + Versioned; impl_versioned_full_protocol_from!( CollatorProtocolMessage, VersionedCollationProtocol, @@ -461,7 +457,7 @@ impl_versioned_try_from!( CollatorProtocolMessage, v1::CollationProtocol::CollatorProtocol(x) => x, v2::CollationProtocol::CollatorProtocol(x) => x, - vstaging::CollationProtocol::CollatorProtocol(x) => x + v2::CollationProtocol::CollatorProtocol(x) => x ); /// v1 notification protocol types. @@ -890,8 +886,8 @@ pub mod vstaging { /// no reason why they can't be change untill vstaging becomes v3 and is released. pub use super::v2::{ declare_signature_payload, BackedCandidateAcknowledgement, BackedCandidateManifest, - BitfieldDistributionMessage, CollationProtocol, CollatorProtocolMessage, - GossipSupportNetworkMessage, StatementDistributionMessage, StatementFilter, + BitfieldDistributionMessage, GossipSupportNetworkMessage, StatementDistributionMessage, + StatementFilter, }; /// Network messages used by the approval distribution subsystem. diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index 0f1d9de2a565..eb483dec9709 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -164,8 +164,6 @@ impl PeerSet { Some("collation/1") } else if version == CollationVersion::V2.into() { Some("collation/2") - } else if version == CollationVersion::VStaging.into() { - Some("collation/3") } else { None }, @@ -241,8 +239,6 @@ pub enum CollationVersion { V1 = 1, /// The second version. V2 = 2, - /// Same format as V2, - VStaging = 3, } /// Marker indicating the version is unknown. From a8d234cab7174ecea0d49650899273fd21024d81 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 16:13:34 +0300 Subject: [PATCH 45/89] Addressed review comments Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 19 ++++++---- polkadot/node/network/protocol/src/lib.rs | 2 +- .../statement-distribution/src/v2/mod.rs | 38 +++++++++++++++++-- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 46d687035f51..47482eef7640 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -1674,8 +1674,7 @@ impl State { .approval_entries(index) .into_iter() .filter_map(|approval_entry| approval_entry.approval(index)) - .map(|approval| (approval.validator, approval.signature)) - .collect::>(); + .map(|approval| (approval.validator, approval.signature)); all_sigs.extend(sigs); } all_sigs @@ -2304,10 +2303,9 @@ pub const MAX_APPROVAL_BATCH_SIZE: usize = ensure_size_not_zero( async fn send_assignments_batched_inner( sender: &mut impl overseer::ApprovalDistributionSenderTrait, batch: impl IntoIterator, - peers: &[PeerId], + peers: Vec, peer_version: ValidationVersion, ) { - let peers = peers.into_iter().cloned().collect::>(); if peer_version == ValidationVersion::VStaging { sender .send_message(NetworkBridgeTxMessage::SendValidationMessage( @@ -2384,15 +2382,20 @@ pub(crate) async fn send_assignments_batched( send_assignments_batched_inner( sender, batch.clone(), - &v1_peers, + v1_peers.clone(), ValidationVersion::V1, ) .await; } if !v2_peers.is_empty() { - send_assignments_batched_inner(sender, batch, &v2_peers, ValidationVersion::V2) - .await; + send_assignments_batched_inner( + sender, + batch, + v2_peers.clone(), + ValidationVersion::V2, + ) + .await; } } } @@ -2405,7 +2408,7 @@ pub(crate) async fn send_assignments_batched( send_assignments_batched_inner( sender, batch, - &vstaging_peers, + vstaging_peers.clone(), ValidationVersion::VStaging, ) .await; diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index a7251f7bf8ed..9aeeb98ea9d6 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -258,7 +258,7 @@ pub enum Versioned { V1(V1), /// V2 type. V2(V2), - /// VStaging type. + /// VStaging type VStaging(VStaging), } diff --git a/polkadot/node/network/statement-distribution/src/v2/mod.rs b/polkadot/node/network/statement-distribution/src/v2/mod.rs index 0f29e33676b5..2f3bf7b38b3e 100644 --- a/polkadot/node/network/statement-distribution/src/v2/mod.rs +++ b/polkadot/node/network/statement-distribution/src/v2/mod.rs @@ -752,13 +752,21 @@ fn pending_statement_network_message( protocol_v2::StatementDistributionMessage::Statement(relay_parent, signed) }) .map(|msg| (vec![peer.0], Versioned::V2(msg).into())), - _ => statement_store + ValidationVersion::VStaging => statement_store .validator_statement(originator, compact) .map(|s| s.as_unchecked().clone()) .map(|signed| { protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, signed) }) .map(|msg| (vec![peer.0], Versioned::VStaging(msg).into())), + ValidationVersion::V1 => { + gum::error!( + target: LOG_TARGET, + "Bug ValidationVersion::V1 should not be used in statement-distribution v2, + legacy should have handled this" + ); + None + }, } } @@ -883,7 +891,7 @@ async fn send_pending_grid_messages( ) .into(), )), - _ => messages.push(( + ValidationVersion::VStaging => messages.push(( vec![peer_id.0], Versioned::VStaging( protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest( @@ -892,6 +900,13 @@ async fn send_pending_grid_messages( ) .into(), )), + ValidationVersion::V1 => { + gum::error!( + target: LOG_TARGET, + "Bug ValidationVersion::V1 should not be used in statement-distribution v2, + legacy should have handled this" + ); + } }; }, grid::ManifestKind::Acknowledgement => { @@ -2183,13 +2198,20 @@ fn post_acknowledgement_statement_messages( ) .into(), )), - _ => messages.push(Versioned::VStaging( + ValidationVersion::VStaging => messages.push(Versioned::VStaging( protocol_vstaging::StatementDistributionMessage::Statement( relay_parent, statement.as_unchecked().clone(), ) .into(), )), + ValidationVersion::V1 => { + gum::error!( + target: LOG_TARGET, + "Bug ValidationVersion::V1 should not be used in statement-distribution v2, + legacy should have handled this" + ); + }, }; } @@ -2324,13 +2346,21 @@ fn acknowledgement_and_statement_messages( let mut messages = match peer.1 { ValidationVersion::V2 => vec![(vec![peer.0], msg_v2.into())], - _ => vec![( + ValidationVersion::VStaging => vec![( vec![peer.0], Versioned::VStaging(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( acknowledgement, )) .into(), )], + ValidationVersion::V1 => { + gum::error!( + target: LOG_TARGET, + "Bug ValidationVersion::V1 should not be used in statement-distribution v2, + legacy should have handled this" + ); + return Vec::new() + }, }; local_validator.grid_tracker.manifest_sent_to( From d728cf26ad600a07700653d2213adbc1f458ad30 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 17:02:38 +0300 Subject: [PATCH 46/89] Make clippy happy Signed-off-by: Alexandru Gheorghe --- .../collator-protocol/src/validator_side/tests/mod.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 55df1c6d18f4..9812998aab76 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -399,12 +399,6 @@ async fn connect_and_declare_collator( para_id, collator.sign(&protocol_v1::declare_signature_payload(&peer)), )), - CollationVersion::VStaging => - Versioned::VStaging(protocol_vstaging::CollatorProtocolMessage::Declare( - collator.public(), - para_id, - collator.sign(&protocol_v1::declare_signature_payload(&peer)), - )), }; overseer_send( From 05ea75d219d47f433cb640efd602f26250381010 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 21:47:51 +0300 Subject: [PATCH 47/89] Build a test image with `network-protocol-staging` Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index fefa3739a9ff..1c1e3bb4b3c8 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker - time ROCOCO_EPOCH_DURATION=10 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/ - time ROCOCO_EPOCH_DURATION=100 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-100/ - time ROCOCO_EPOCH_DURATION=600 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-600/ From 261285f91a43560c1f7874a26f8789d033975db7 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Oct 2023 09:54:51 +0300 Subject: [PATCH 48/89] Enable assignments Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 1f751e2bf140..db74302f0225 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -278,7 +278,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) } fn check_assignment_cert( From db9c3a94ab44ae2eef2e755e0e463811e91b049b Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Oct 2023 12:05:10 +0300 Subject: [PATCH 49/89] Revert "Enable assignments v2 used for testing" This reverts commit 261285f91a43560c1f7874a26f8789d033975db7. --- .gitlab/pipeline/build.yml | 2 +- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 1c1e3bb4b3c8..fefa3739a9ff 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker - time ROCOCO_EPOCH_DURATION=10 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/ - time ROCOCO_EPOCH_DURATION=100 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-100/ - time ROCOCO_EPOCH_DURATION=600 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-600/ diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index db74302f0225..1f751e2bf140 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -278,7 +278,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) } fn check_assignment_cert( From 15eaa0a20f67d3d7569eda39e0ece1bfee8ca6ec Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 26 Oct 2023 11:01:32 +0300 Subject: [PATCH 50/89] Cleanup un-needed dependency to test-helpers Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 1 + polkadot/node/core/approval-voting/Cargo.toml | 2 -- .../src/approval_db/v2/migration_helpers.rs | 13 ++++++++----- polkadot/node/service/Cargo.toml | 1 + polkadot/node/service/src/parachains_db/upgrade.rs | 4 +++- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22ee0eff9e1c..a096faf19a3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12892,6 +12892,7 @@ dependencies = [ "polkadot-overseer", "polkadot-parachain-primitives", "polkadot-primitives", + "polkadot-primitives-test-helpers", "polkadot-rpc", "polkadot-runtime-parachains", "polkadot-statement-distribution", diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index 5dd8cc1714f0..b89d7d2cedc0 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -31,8 +31,6 @@ sp-consensus = { path = "../../../../substrate/primitives/consensus/common", def sp-consensus-slots = { path = "../../../../substrate/primitives/consensus/slots", default-features = false } sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto", default-features = false, features = ["full_crypto"] } sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -# Needed for migration test helpers -test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } rand_core = "0.5.1" [dev-dependencies] diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs index a70d766fc8c9..74e997c7af84 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -21,10 +21,9 @@ use polkadot_node_primitives::approval::v1::{ AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, }; use polkadot_node_subsystem_util::database::Database; +use sp_application_crypto::sp_core::H256; use std::{collections::HashSet, sync::Arc}; -use ::test_helpers::dummy_candidate_receipt; - fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); let msg = b"test-garbage"; @@ -145,10 +144,14 @@ pub fn v1_to_v2_sanity_check( } // Fills the db with dummy data in v1 scheme. -pub fn v1_to_v2_fill_test_data( +pub fn v1_to_v2_fill_test_data( db: Arc, config: Config, -) -> Result> { + dummy_candidate_create: F, +) -> Result> +where + F: Fn(H256) -> CandidateReceipt, +{ let mut backend = crate::DbBackend::new(db.clone(), config); let mut overlay_db = crate::OverlayedBackend::new(&backend); let mut expected_candidates = HashSet::new(); @@ -161,7 +164,7 @@ pub fn v1_to_v2_fill_test_data( for relay_number in 1..=RELAY_BLOCK_COUNT { let relay_hash = Hash::repeat_byte(relay_number as u8); let assignment_core_index = CoreIndex(relay_number); - let candidate = dummy_candidate_receipt(relay_hash); + let candidate = dummy_candidate_create(relay_hash); let candidate_hash = candidate.hash(); let at_height = vec![relay_hash]; diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 6d36c3674028..133a1094c4ca 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -139,6 +139,7 @@ polkadot-statement-distribution = { path = "../network/statement-distribution", [dev-dependencies] polkadot-test-client = { path = "../test/client" } polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } +test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../primitives/test-helpers" } env_logger = "0.9.0" assert_matches = "1.5.0" serial_test = "2.0.0" diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index 78e53a251eed..1d76c79d3e32 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -442,6 +442,7 @@ mod tests { *, }; use polkadot_node_core_approval_voting::approval_db::v2::migration_helpers::v1_to_v2_fill_test_data; + use test_helpers::dummy_candidate_receipt; #[test] fn test_paritydb_migrate_0_to_1() { @@ -599,7 +600,8 @@ mod tests { assert_eq!(db.num_columns(), super::columns::v3::NUM_COLUMNS as u32); let db = DbAdapter::new(db, columns::v3::ORDERED_COL); // Fill the approval voting column with test data. - v1_to_v2_fill_test_data(std::sync::Arc::new(db), approval_cfg).unwrap() + v1_to_v2_fill_test_data(std::sync::Arc::new(db), approval_cfg, dummy_candidate_receipt) + .unwrap() }; try_upgrade_db(&db_dir.path(), DatabaseKind::RocksDB, 4).unwrap(); From be9dddd8676f9af77fe42d3a7ccf684679a2517a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 26 Oct 2023 17:29:03 +0300 Subject: [PATCH 51/89] Address review findings Signed-off-by: Alexandru Gheorghe --- .../approval-voting/src/approval_checking.rs | 125 +++++++++++------- polkadot/node/core/approval-voting/src/lib.rs | 109 +++++++++++---- polkadot/node/core/runtime-api/src/cache.rs | 2 +- 3 files changed, 155 insertions(+), 81 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index 374b55aa0b4f..f1f3b2079220 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -25,6 +25,15 @@ use crate::{ time::Tick, }; +/// Result of counting the necessary tranches needed for approving a block. +#[derive(Debug, PartialEq, Clone)] +pub struct TranchesToApproveResult { + /// The required tranches for approving this block + pub required_tranches: RequiredTranches, + /// The total number of no_shows at the moment we are doing the counting. + pub total_observed_no_shows: usize, +} + /// The required tranches of assignments needed to determine whether a candidate is approved. #[derive(Debug, PartialEq, Clone)] pub enum RequiredTranches { @@ -178,6 +187,7 @@ struct State { next_no_show: Option, /// The last tick at which a considered assignment was received. last_assignment_tick: Option, + total_observed_no_shows: usize, } impl State { @@ -187,41 +197,53 @@ impl State { needed_approvals: usize, n_validators: usize, no_show_duration: Tick, - ) -> RequiredTranches { + ) -> TranchesToApproveResult { let covering = if self.depth == 0 { 0 } else { self.covering }; if self.depth != 0 && self.assignments + covering + self.uncovered >= n_validators { - return RequiredTranches::All + return TranchesToApproveResult { + required_tranches: RequiredTranches::All, + total_observed_no_shows: self.total_observed_no_shows, + } } // If we have enough assignments and all no-shows are covered, we have reached the number // of tranches that we need to have. if self.assignments >= needed_approvals && (covering + self.uncovered) == 0 { - return RequiredTranches::Exact { - needed: tranche, - tolerated_missing: self.covered, - next_no_show: self.next_no_show, - last_assignment_tick: self.last_assignment_tick, + return TranchesToApproveResult { + required_tranches: RequiredTranches::Exact { + needed: tranche, + tolerated_missing: self.covered, + next_no_show: self.next_no_show, + last_assignment_tick: self.last_assignment_tick, + }, + total_observed_no_shows: self.total_observed_no_shows, } } // We're pending more assignments and should look at more tranches. let clock_drift = self.clock_drift(no_show_duration); if self.depth == 0 { - RequiredTranches::Pending { - considered: tranche, - next_no_show: self.next_no_show, - // during the initial assignment-gathering phase, we want to accept assignments - // from any tranche. Note that honest validators will still not broadcast their - // assignment until it is time to do so, regardless of this value. - maximum_broadcast: DelayTranche::max_value(), - clock_drift, + TranchesToApproveResult { + required_tranches: RequiredTranches::Pending { + considered: tranche, + next_no_show: self.next_no_show, + // during the initial assignment-gathering phase, we want to accept assignments + // from any tranche. Note that honest validators will still not broadcast their + // assignment until it is time to do so, regardless of this value. + maximum_broadcast: DelayTranche::max_value(), + clock_drift, + }, + total_observed_no_shows: self.total_observed_no_shows, } } else { - RequiredTranches::Pending { - considered: tranche, - next_no_show: self.next_no_show, - maximum_broadcast: tranche + (covering + self.uncovered) as DelayTranche, - clock_drift, + TranchesToApproveResult { + required_tranches: RequiredTranches::Pending { + considered: tranche, + next_no_show: self.next_no_show, + maximum_broadcast: tranche + (covering + self.uncovered) as DelayTranche, + clock_drift, + }, + total_observed_no_shows: self.total_observed_no_shows, } } } @@ -276,6 +298,7 @@ impl State { uncovered, next_no_show, last_assignment_tick, + total_observed_no_shows: self.total_observed_no_shows + new_no_shows, } } } @@ -372,22 +395,20 @@ pub fn tranches_to_approve( block_tick: Tick, no_show_duration: Tick, needed_approvals: usize, -) -> (RequiredTranches, usize) { +) -> TranchesToApproveResult { let tick_now = tranche_now as Tick + block_tick; let n_validators = approval_entry.n_validators(); - let initial_state = ( - State { - assignments: 0, - depth: 0, - covered: 0, - covering: needed_approvals, - uncovered: 0, - next_no_show: None, - last_assignment_tick: None, - }, - 0usize, - ); + let initial_state = State { + assignments: 0, + depth: 0, + covered: 0, + covering: needed_approvals, + uncovered: 0, + next_no_show: None, + last_assignment_tick: None, + total_observed_no_shows: 0, + }; // The `ApprovalEntry` doesn't have any data for empty tranches. We still want to iterate over // these empty tranches, so we create an iterator to fill the gaps. @@ -398,7 +419,7 @@ pub fn tranches_to_approve( tranches_with_gaps_filled .scan(Some(initial_state), |state, (tranche, assignments)| { // The `Option` here is used for early exit. - let (s, prev_no_shows) = state.take()?; + let s = state.take()?; let clock_drift = s.clock_drift(no_show_duration); let drifted_tick_now = tick_now.saturating_sub(clock_drift); @@ -437,7 +458,7 @@ pub fn tranches_to_approve( let s = s.advance(n_assignments, no_shows, next_no_show, last_assignment_tick); let output = s.output(tranche, needed_approvals, n_validators, no_show_duration); - *state = match output { + *state = match output.required_tranches { RequiredTranches::Exact { .. } | RequiredTranches::All => { // Wipe the state clean so the next iteration of this closure will terminate // the iterator. This guarantees that we can call `last` further down to see @@ -447,11 +468,11 @@ pub fn tranches_to_approve( RequiredTranches::Pending { .. } => { // Pending results are only interesting when they are the last result of the iterator // i.e. we never achieve a satisfactory level of assignment. - Some((s, no_shows + prev_no_shows)) + Some(s) } }; - Some((output, no_shows + prev_no_shows)) + Some(output) }) .last() .expect("the underlying iterator is infinite, starts at 0, and never exits early before tranche 1; qed") @@ -685,7 +706,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -726,7 +747,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 2, next_no_show: Some(block_tick + no_show_duration), @@ -771,7 +792,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 11, next_no_show: None, @@ -820,7 +841,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -840,7 +861,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 1, next_no_show: None, @@ -894,7 +915,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 1, tolerated_missing: 0, @@ -914,7 +935,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -934,7 +955,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -988,7 +1009,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 2, tolerated_missing: 1, @@ -1011,7 +1032,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 2, next_no_show: None, @@ -1033,7 +1054,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Exact { needed: 3, tolerated_missing: 2, @@ -1091,7 +1112,7 @@ mod tests { no_show_duration, needed_approvals, ) - .0, + .required_tranches, RequiredTranches::Pending { considered: 10, next_no_show: None, @@ -1368,10 +1389,11 @@ mod tests { uncovered: 0, next_no_show: None, last_assignment_tick: None, + total_observed_no_shows: 0, }; assert_eq!( - state.output(0, 10, 10, 20), + state.output(0, 10, 10, 20).required_tranches, RequiredTranches::Pending { considered: 0, next_no_show: None, @@ -1391,10 +1413,11 @@ mod tests { uncovered: 0, next_no_show: None, last_assignment_tick: None, + total_observed_no_shows: 0, }; assert_eq!( - state.output(0, 10, 10, 20), + state.output(0, 10, 10, 20).required_tranches, RequiredTranches::Exact { needed: 0, tolerated_missing: 0, diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index ffa7cb8b6a98..e21ad331431d 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -99,7 +99,7 @@ mod persisted_entries; mod time; use crate::{ - approval_checking::Check, + approval_checking::{Check, TranchesToApproveResult}, approval_db::v2::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, criteria::InvalidAssignmentReason, @@ -839,17 +839,22 @@ impl State { ); if let Some(approval_entry) = candidate_entry.approval_entry(&block_hash) { - let (required_tranches, last_no_shows) = approval_checking::tranches_to_approve( - approval_entry, - candidate_entry.approvals(), - tranche_now, - block_tick, - no_show_duration, - session_info.needed_approvals as _, - ); + let TranchesToApproveResult { required_tranches, total_observed_no_shows } = + approval_checking::tranches_to_approve( + approval_entry, + candidate_entry.approvals(), + tranche_now, + block_tick, + no_show_duration, + session_info.needed_approvals as _, + ); - let status = - ApprovalStatus { required_tranches, block_tick, tranche_now, last_no_shows }; + let status = ApprovalStatus { + required_tranches, + block_tick, + tranche_now, + last_no_shows: total_observed_no_shows, + }; Some((approval_entry, status)) } else { @@ -887,9 +892,18 @@ impl State { .map(|cache| cache.insert(block_hash, params)); params }, - _ => { + Ok(Err(err)) => { + gum::error!( + target: LOG_TARGET, + ?err, + "Could not request approval voting params from runtime using defaults" + ); + ApprovalVotingParams { max_approval_coalesce_count: 1 } + }, + Err(err) => { gum::error!( target: LOG_TARGET, + ?err, "Could not request approval voting params from runtime using defaults" ); ApprovalVotingParams { max_approval_coalesce_count: 1 } @@ -1764,13 +1778,34 @@ async fn get_approval_signatures_for_candidate( let votes = votes .into_iter() .map(|(validator_index, (hash, signed_candidates_indices, signature))| { - let candidates_hashes = - candidate_indices_to_candidate_hashes.get(&hash).expect("Can't fail because it is the same hash we sent to approval-distribution; qed"); + let candidates_hashes = candidate_indices_to_candidate_hashes.get(&hash); + + if candidates_hashes.is_none() { + gum::warn!( + target: LOG_TARGET, + ?hash, + "Possible bug! Could not find map of candidate_hashes for block hash received from approval-distribution" + ); + } + let signed_candidates_hashes: Vec = signed_candidates_indices .into_iter() - .map(|candidate_index| { - *(candidates_hashes.get(&candidate_index).expect("Can't fail because we already checked the signature was valid, so we should be able to find the hash; qed")) + .filter_map(|candidate_index| { + candidates_hashes.and_then(|candidate_hashes| { + if let Some(candidate_hash) = + candidate_hashes.get(&candidate_index) + { + Some(*candidate_hash) + } else { + gum::warn!( + target: LOG_TARGET, + ?candidate_index, + "Possible bug! Could not find candidate hash for candidate_index coming from approval-distribution" + ); + None + } + }) }) .collect(); (validator_index, (signed_candidates_hashes, signature)) @@ -2512,7 +2547,13 @@ where ) .check_signature( &pubkey, - *candidate_hashes.first().expect("Checked above this is not empty; qed"), + if let Some(candidate_hash) = candidate_hashes.first() { + *candidate_hash + } else { + respond_early!(ApprovalCheckResult::Bad(ApprovalCheckError::InvalidValidatorIndex( + approval.validator + ),)) + }, block_entry.session(), &approval.signature, ) { @@ -2880,7 +2921,7 @@ async fn process_wakeup( None => return Ok(Vec::new()), }; - let (tranches_to_approve, _last_no_shows) = approval_checking::tranches_to_approve( + let tranches_to_approve = approval_checking::tranches_to_approve( &approval_entry, candidate_entry.approvals(), tranche_now, @@ -2892,7 +2933,7 @@ async fn process_wakeup( let should_trigger = should_trigger_assignment( &approval_entry, &candidate_entry, - tranches_to_approve, + tranches_to_approve.required_tranches, tranche_now, ); @@ -3471,17 +3512,27 @@ async fn maybe_create_signature( .map(|candidate_hash| db.load_candidate_entry(candidate_hash)) .collect::>>>()?; - for candidate_entry in candidate_entries { - let mut candidate_entry = candidate_entry - .expect("Candidate was scheduled to be signed entry in db should exist; qed"); - let approval_entry = candidate_entry - .approval_entry_mut(&block_entry.block_hash()) - .expect("Candidate was scheduled to be signed entry in db should exist; qed"); - approval_entry.import_approval_sig(OurApproval { - signature: signature.clone(), - signed_candidates_indices: candidates_indices.clone(), + for mut candidate_entry in candidate_entries { + // let mut candidate_entry = candidate_entry + // .expect("Candidate was scheduled to be signed entry in db should exist; qed"); + let approval_entry = candidate_entry.as_mut().and_then(|candidate_entry| { + candidate_entry.approval_entry_mut(&block_entry.block_hash()) }); - db.write_candidate_entry(candidate_entry); + + match approval_entry { + Some(approval_entry) => approval_entry.import_approval_sig(OurApproval { + signature: signature.clone(), + signed_candidates_indices: candidates_indices.clone(), + }), + None => { + gum::error!( + target: LOG_TARGET, + candidate_entry = ?candidate_entry, + "Candidate scheduled for signing approval entry should not be None" + ); + }, + }; + candidate_entry.map(|candidate_entry| db.write_candidate_entry(candidate_entry)); } metrics.on_approval_produced(); diff --git a/polkadot/node/core/runtime-api/src/cache.rs b/polkadot/node/core/runtime-api/src/cache.rs index 1996578dd4b6..83ebf6884890 100644 --- a/polkadot/node/core/runtime-api/src/cache.rs +++ b/polkadot/node/core/runtime-api/src/cache.rs @@ -61,13 +61,13 @@ pub(crate) struct RequestResultCache { LruMap<(Hash, ParaId, OccupiedCoreAssumption), Option>, version: LruMap, disputes: LruMap)>>, - approval_voting_params: LruMap, unapplied_slashes: LruMap>, key_ownership_proof: LruMap<(Hash, ValidatorId), Option>, minimum_backing_votes: LruMap, disabled_validators: LruMap>, para_backing_state: LruMap<(Hash, ParaId), Option>, async_backing_params: LruMap, + approval_voting_params: LruMap, } impl Default for RequestResultCache { From 2385fad585fca15357bc01c5f36cbc1f67d35bfa Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 26 Oct 2023 18:21:05 +0300 Subject: [PATCH 52/89] Address review findings Signed-off-by: Alexandru Gheorghe --- .../0007-approval-voting-coalescing.toml | 85 +------------------ .../0007-approval-voting-coalescing.zndsl | 42 ++------- 2 files changed, 10 insertions(+), 117 deletions(-) diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index c7eeeb5383e0..872bf025de97 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -17,89 +17,10 @@ chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default limits = { memory = "4G", cpu = "2" } requests = { memory = "2G", cpu = "1" } - [[relaychain.nodes]] + [[relaychain.node_groups]] name = "alice" - args = [ "--alice", "-lparachain=trace,runtime=debug" ] - - [[relaychain.nodes]] - name = "bob" - args = [ "--bob", "-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "charlie" - args = [ "--charlie", "-lparachain=debug,runtime=debug" ] - - [[relaychain.nodes]] - name = "dave" - args = [ "--dave", "-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "ferdie" - args = [ "--ferdie", "-lparachain=debug,runtime=debug" ] - - [[relaychain.nodes]] - name = "eve" - args = [ "--eve", "-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "one" - args = [ "--one", "-lparachain=debug,runtime=debug" ] - - [[relaychain.nodes]] - name = "two" - args = [ "--two", "-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "nine" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "eleven" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "twelve" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "thirteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "fourteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "fifteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "sixteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "seventeen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "eithteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "nineteen" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "twenty" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "twentyone" - args = ["-lparachain=debug,runtime=debug"] - - [[relaychain.nodes]] - name = "twentytwo" - args = ["-lparachain=debug,runtime=debug"] + args = [ "-lparachain=trace,runtime=debug" ] + count = 21 [[parachains]] id = 2000 diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl index a4cc454f38cc..07ef30d546a1 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl @@ -4,23 +4,16 @@ Creds: config # Check authority status. alice: reports node_roles is 4 -bob: reports node_roles is 4 -charlie: reports node_roles is 4 -dave: reports node_roles is 4 -eve: reports node_roles is 4 -ferdie: reports node_roles is 4 -one: reports node_roles is 4 -two: reports node_roles is 4 # Ensure parachains are registered. alice: parachain 2000 is registered within 60 seconds -bob: parachain 2001 is registered within 60 seconds -charlie: parachain 2002 is registered within 60 seconds -dave: parachain 2003 is registered within 60 seconds -ferdie: parachain 2004 is registered within 60 seconds -eve: parachain 2005 is registered within 60 seconds -one: parachain 2006 is registered within 60 seconds -two: parachain 2007 is registered within 60 seconds +alice: parachain 2001 is registered within 60 seconds +alice: parachain 2002 is registered within 60 seconds +alice: parachain 2003 is registered within 60 seconds +alice: parachain 2004 is registered within 60 seconds +alice: parachain 2005 is registered within 60 seconds +alice: parachain 2006 is registered within 60 seconds +alice: parachain 2007 is registered within 60 seconds # Ensure parachains made progress. alice: parachain 2000 block height is at least 10 within 300 seconds @@ -33,28 +26,7 @@ alice: parachain 2006 block height is at least 10 within 300 seconds alice: parachain 2007 block height is at least 10 within 300 seconds alice: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -bob: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -charlie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -dave: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -eve: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -ferdie: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -one: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds -two: reports substrate_block_height{status="finalized"} is at least 30 within 400 seconds alice: reports polkadot_parachain_approval_checking_finality_lag < 3 -bob: reports polkadot_parachain_approval_checking_finality_lag < 3 -charlie: reports polkadot_parachain_approval_checking_finality_lag < 3 -dave: reports polkadot_parachain_approval_checking_finality_lag < 3 -ferdie: reports polkadot_parachain_approval_checking_finality_lag < 3 -eve: reports polkadot_parachain_approval_checking_finality_lag < 3 -one: reports polkadot_parachain_approval_checking_finality_lag < 3 -two: reports polkadot_parachain_approval_checking_finality_lag < 3 alice: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -bob: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -charlie: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -dave: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -ferdie: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -eve: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -one: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds -two: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds \ No newline at end of file From a2c032002502005fe3047e50560bfa235801e00d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 27 Oct 2023 15:53:59 +0300 Subject: [PATCH 53/89] Add new approval_db version ... so that we can decouple this PR from https://github.com/paritytech/polkadot-sdk/pull/1178 and be able to merge them separately. Signed-off-by: Alexandru Gheorghe --- .../approval-voting/src/approval_checking.rs | 36 +- .../approval_db/common/migration_helpers.rs | 36 ++ .../src/approval_db/common/mod.rs | 293 +++++++++ .../approval-voting/src/approval_db/mod.rs | 2 + .../src/approval_db/v2/migration_helpers.rs | 64 +- .../approval-voting/src/approval_db/v2/mod.rs | 277 +-------- .../src/approval_db/v2/tests.rs | 66 +- .../src/approval_db/v3/migration_helpers.rs | 237 +++++++ .../approval-voting/src/approval_db/v3/mod.rs | 139 +++++ .../src/approval_db/v3/tests.rs | 580 ++++++++++++++++++ .../node/core/approval-voting/src/backend.rs | 15 +- .../node/core/approval-voting/src/import.rs | 22 +- polkadot/node/core/approval-voting/src/lib.rs | 6 +- polkadot/node/core/approval-voting/src/ops.rs | 2 +- .../approval-voting/src/persisted_entries.rs | 94 ++- .../node/core/approval-voting/src/tests.rs | 4 +- .../node/service/src/parachains_db/upgrade.rs | 99 ++- 17 files changed, 1543 insertions(+), 429 deletions(-) create mode 100644 polkadot/node/core/approval-voting/src/approval_db/common/migration_helpers.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/common/mod.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs create mode 100644 polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs diff --git a/polkadot/node/core/approval-voting/src/approval_checking.rs b/polkadot/node/core/approval-voting/src/approval_checking.rs index f1f3b2079220..0aa6102fbd6d 100644 --- a/polkadot/node/core/approval-voting/src/approval_checking.rs +++ b/polkadot/node/core/approval-voting/src/approval_checking.rs @@ -498,7 +498,7 @@ mod tests { 0, ); - let approval_entry = approval_db::v2::ApprovalEntry { + let approval_entry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: BitVec::default(), our_assignment: None, @@ -537,17 +537,17 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v2::ApprovalEntry { + let approval_entry = approval_db::v3::ApprovalEntry { tranches: vec![ - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 0, assignments: (0..2).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 2, assignments: (5..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, @@ -611,17 +611,17 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v2::ApprovalEntry { + let approval_entry = approval_db::v3::ApprovalEntry { tranches: vec![ - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 0, assignments: (0..4).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 1, assignments: (4..6).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 2, assignments: (6..10).map(|i| (ValidatorIndex(i), 0.into())).collect(), }, @@ -677,7 +677,7 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; 5], our_assignment: None, @@ -722,7 +722,7 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, @@ -763,7 +763,7 @@ mod tests { let no_show_duration = 10; let needed_approvals = 4; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; 10], our_assignment: None, @@ -809,7 +809,7 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, @@ -878,7 +878,7 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, @@ -972,7 +972,7 @@ mod tests { let needed_approvals = 4; let n_validators = 8; - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), assigned_validators: bitvec![u8, BitOrderLsb0; 0; n_validators], our_assignment: None, @@ -1084,10 +1084,10 @@ mod tests { candidate.mark_approval(ValidatorIndex(i)); } - let approval_entry = approval_db::v2::ApprovalEntry { + let approval_entry = approval_db::v3::ApprovalEntry { tranches: vec![ // Assignments with invalid validator indexes. - approval_db::v2::TrancheEntry { + approval_db::v3::TrancheEntry { tranche: 1, assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), }, @@ -1138,7 +1138,7 @@ mod tests { ]; for test_tranche in test_tranches { - let mut approval_entry: ApprovalEntry = approval_db::v2::ApprovalEntry { + let mut approval_entry: ApprovalEntry = approval_db::v3::ApprovalEntry { tranches: Vec::new(), backing_group: GroupIndex(0), our_assignment: None, diff --git a/polkadot/node/core/approval-voting/src/approval_db/common/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/common/migration_helpers.rs new file mode 100644 index 000000000000..32ddbaf81277 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/common/migration_helpers.rs @@ -0,0 +1,36 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; + +use polkadot_node_primitives::approval::v1::{ + AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, +}; + +pub fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +pub fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { + let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); + let msg = b"test-garbage"; + let mut prng = rand_core::OsRng; + let keypair = schnorrkel::Keypair::generate_with(&mut prng); + let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); + let out = inout.to_output(); + + AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/common/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/common/mod.rs new file mode 100644 index 000000000000..249dcf912df5 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/common/mod.rs @@ -0,0 +1,293 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Common helper functions for all versions of approval-voting database. +use std::sync::Arc; + +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; +use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_primitives::{BlockNumber, CandidateHash, CandidateIndex, Hash}; + +use crate::{ + backend::{Backend, BackendWriteOp, V1ReadBackend, V2ReadBackend}, + persisted_entries, +}; + +use super::{ + v2::{load_block_entry_v1, load_candidate_entry_v1}, + v3::{load_block_entry_v2, load_candidate_entry_v2, BlockEntry, CandidateEntry}, +}; + +pub mod migration_helpers; + +const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; + +/// A range from earliest..last block number stored within the DB. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); +/// The database config. +#[derive(Debug, Clone, Copy)] +pub struct Config { + /// The column family in the database where data is stored. + pub col_approval_data: u32, +} + +/// `DbBackend` is a concrete implementation of the higher-level Backend trait +pub struct DbBackend { + inner: Arc, + config: Config, +} + +impl DbBackend { + /// Create a new [`DbBackend`] with the supplied key-value store and + /// config. + pub fn new(db: Arc, config: Config) -> Self { + DbBackend { inner: db, config } + } +} + +/// Errors while accessing things from the DB. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum Error { + Io(std::io::Error), + InvalidDecoding(parity_scale_codec::Error), + InternalError(SubsystemError), +} + +impl std::error::Error for Error {} + +/// Result alias for DB errors. +pub type Result = std::result::Result; + +impl Backend for DbBackend { + fn load_block_entry( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } + + fn load_candidate_entry( + &self, + candidate_hash: &CandidateHash, + ) -> SubsystemResult> { + load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) + } + + fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { + load_blocks_at_height(&*self.inner, &self.config, block_height) + } + + fn load_all_blocks(&self) -> SubsystemResult> { + load_all_blocks(&*self.inner, &self.config) + } + + fn load_stored_blocks(&self) -> SubsystemResult> { + load_stored_blocks(&*self.inner, &self.config) + } + + /// Atomically write the list of operations, with later operations taking precedence over prior. + fn write(&mut self, ops: I) -> SubsystemResult<()> + where + I: IntoIterator, + { + let mut tx = DBTransaction::new(); + for op in ops { + match op { + BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { + tx.put_vec( + self.config.col_approval_data, + &STORED_BLOCKS_KEY, + stored_block_range.encode(), + ); + }, + BackendWriteOp::DeleteStoredBlockRange => { + tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); + }, + BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { + tx.put_vec( + self.config.col_approval_data, + &blocks_at_height_key(h), + blocks.encode(), + ); + }, + BackendWriteOp::DeleteBlocksAtHeight(h) => { + tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); + }, + BackendWriteOp::WriteBlockEntry(block_entry) => { + let block_entry: BlockEntry = block_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + }, + BackendWriteOp::DeleteBlockEntry(hash) => { + tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); + }, + BackendWriteOp::WriteCandidateEntry(candidate_entry) => { + let candidate_entry: CandidateEntry = candidate_entry.into(); + tx.put_vec( + self.config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + }, + BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { + tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); + }, + } + } + + self.inner.write(tx).map_err(|e| e.into()) + } +} + +impl V1ReadBackend for DbBackend { + fn load_candidate_entry_v1( + &self, + candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, + ) -> SubsystemResult> { + load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) + .map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v1(e, candidate_index))) + } + + fn load_block_entry_v1( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry_v1(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } +} + +impl V2ReadBackend for DbBackend { + fn load_candidate_entry_v2( + &self, + candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, + ) -> SubsystemResult> { + load_candidate_entry_v2(&*self.inner, &self.config, candidate_hash) + .map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v2(e, candidate_index))) + } + + fn load_block_entry_v2( + &self, + block_hash: &Hash, + ) -> SubsystemResult> { + load_block_entry_v2(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) + } +} + +pub(crate) fn load_decode( + store: &dyn Database, + col_approval_data: u32, + key: &[u8], +) -> Result> { + match store.get(col_approval_data, key)? { + None => Ok(None), + Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), + } +} + +/// The key a given block entry is stored under. +pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { + const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(block_hash.as_ref()); + + key +} + +/// The key a given candidate entry is stored under. +pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { + const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; + + let mut key = [0u8; 14 + 32]; + key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); + key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); + + key +} + +/// The key a set of block hashes corresponding to a block number is stored under. +pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { + const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; + + let mut key = [0u8; 12 + 4]; + key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); + block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); + + key +} + +/// Return all blocks which have entries in the DB, ascending, by height. +pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { + let mut hashes = Vec::new(); + if let Some(stored_blocks) = load_stored_blocks(store, config)? { + for height in stored_blocks.0..stored_blocks.1 { + let blocks = load_blocks_at_height(store, config, &height)?; + hashes.extend(blocks); + } + } + + Ok(hashes) +} + +/// Load the stored-blocks key from the state. +pub fn load_stored_blocks( + store: &dyn Database, + config: &Config, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a blocks-at-height entry for a given block number. +pub fn load_blocks_at_height( + store: &dyn Database, + config: &Config, + block_number: &BlockNumber, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) + .map(|x| x.unwrap_or_default()) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store. +pub fn load_block_entry( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a candidate entry from the aux store in current version format. +pub fn load_candidate_entry( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/mod.rs index 20fb6aa82d8d..78942a507f4b 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/mod.rs @@ -30,5 +30,7 @@ //! In the future, we may use a temporary DB which doesn't need to be wiped, but for the //! time being we share the same DB with the rest of Substrate. +pub mod common; pub mod v1; pub mod v2; +pub mod v3; diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs index e123abf59419..df6e4754dbd6 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/migration_helpers.rs @@ -16,25 +16,19 @@ //! Approval DB migration helpers. use super::*; -use crate::backend::Backend; -use polkadot_node_primitives::approval::v1::{ - AssignmentCert, AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, +use crate::{ + approval_db::common::{ + migration_helpers::{dummy_assignment_cert, make_bitvec}, + Error, Result, StoredBlockRange, + }, + backend::Backend, }; + +use polkadot_node_primitives::approval::v1::AssignmentCertKind; use polkadot_node_subsystem_util::database::Database; use sp_application_crypto::sp_core::H256; use std::{collections::HashSet, sync::Arc}; -fn dummy_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { - let ctx = schnorrkel::signing_context(RELAY_VRF_MODULO_CONTEXT); - let msg = b"test-garbage"; - let mut prng = rand_core::OsRng; - let keypair = schnorrkel::Keypair::generate_with(&mut prng); - let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); - let out = inout.to_output(); - - AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } -} - fn make_block_entry_v1( block_hash: Hash, parent_hash: Hash, @@ -54,14 +48,10 @@ fn make_block_entry_v1( } } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - /// Migrates `OurAssignment`, `CandidateEntry` and `ApprovalEntry` to version 2. /// Returns on any error. /// Must only be used in parachains DB migration code - `polkadot-service` crate. -pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { +pub fn v1_to_latest(db: Arc, config: Config) -> Result<()> { let mut backend = crate::DbBackend::new(db, config); let all_blocks = backend .load_all_blocks() @@ -111,42 +101,8 @@ pub fn v1_to_v2(db: Arc, config: Config) -> Result<()> { Ok(()) } -// Checks if the migration doesn't leave the DB in an unsane state. -// This function is to be used in tests. -pub fn v1_to_v2_sanity_check( - db: Arc, - config: Config, - expected_candidates: HashSet, -) -> Result<()> { - let backend = crate::DbBackend::new(db, config); - - let all_blocks = backend - .load_all_blocks() - .unwrap() - .iter() - .map(|block_hash| backend.load_block_entry(block_hash).unwrap().unwrap()) - .collect::>(); - - let mut candidates = HashSet::new(); - - // Iterate all blocks and approval entries. - for block in all_blocks { - for (_core_index, candidate_hash) in block.candidates() { - // Loading the candidate will also perform the conversion to the updated format and - // return that represantation. - if let Some(candidate_entry) = backend.load_candidate_entry(&candidate_hash).unwrap() { - candidates.insert(candidate_entry.candidate.hash()); - } - } - } - - assert_eq!(candidates, expected_candidates); - - Ok(()) -} - // Fills the db with dummy data in v1 scheme. -pub fn v1_to_v2_fill_test_data( +pub fn v1_fill_test_data( db: Arc, config: Config, dummy_candidate_create: F, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs index 2846b0ca499b..da42fc5be485 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/mod.rs @@ -17,10 +17,7 @@ //! Version 2 of the DB schema. use parity_scale_codec::{Decode, Encode}; -use polkadot_node_primitives::approval::{ - v1::DelayTranche, - v2::{AssignmentCertV2, CandidateBitfield}, -}; +use polkadot_node_primitives::approval::{v1::DelayTranche, v2::AssignmentCertV2}; use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; use polkadot_node_subsystem_util::database::{DBTransaction, Database}; use polkadot_primitives::{ @@ -31,139 +28,16 @@ use polkadot_primitives::{ use sp_consensus_slots::Slot; use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; -use crate::{ - backend::{Backend, BackendWriteOp, V1ReadBackend}, - persisted_entries, -}; +use crate::backend::V1ReadBackend; -const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks"; +use super::common::{block_entry_key, candidate_entry_key, load_decode, Config}; pub mod migration_helpers; #[cfg(test)] pub mod tests; -/// `DbBackend` is a concrete implementation of the higher-level Backend trait -pub struct DbBackend { - inner: Arc, - config: Config, -} - -impl DbBackend { - /// Create a new [`DbBackend`] with the supplied key-value store and - /// config. - pub fn new(db: Arc, config: Config) -> Self { - DbBackend { inner: db, config } - } -} - -impl V1ReadBackend for DbBackend { - fn load_candidate_entry_v1( - &self, - candidate_hash: &CandidateHash, - candidate_index: CandidateIndex, - ) -> SubsystemResult> { - load_candidate_entry_v1(&*self.inner, &self.config, candidate_hash) - .map(|e| e.map(|e| persisted_entries::CandidateEntry::from_v1(e, candidate_index))) - } - - fn load_block_entry_v1( - &self, - block_hash: &Hash, - ) -> SubsystemResult> { - load_block_entry_v1(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) - } -} - -impl Backend for DbBackend { - fn load_block_entry( - &self, - block_hash: &Hash, - ) -> SubsystemResult> { - load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into)) - } - - fn load_candidate_entry( - &self, - candidate_hash: &CandidateHash, - ) -> SubsystemResult> { - load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into)) - } - - fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult> { - load_blocks_at_height(&*self.inner, &self.config, block_height) - } - - fn load_all_blocks(&self) -> SubsystemResult> { - load_all_blocks(&*self.inner, &self.config) - } - - fn load_stored_blocks(&self) -> SubsystemResult> { - load_stored_blocks(&*self.inner, &self.config) - } - - /// Atomically write the list of operations, with later operations taking precedence over prior. - fn write(&mut self, ops: I) -> SubsystemResult<()> - where - I: IntoIterator, - { - let mut tx = DBTransaction::new(); - for op in ops { - match op { - BackendWriteOp::WriteStoredBlockRange(stored_block_range) => { - tx.put_vec( - self.config.col_approval_data, - &STORED_BLOCKS_KEY, - stored_block_range.encode(), - ); - }, - BackendWriteOp::DeleteStoredBlockRange => { - tx.delete(self.config.col_approval_data, &STORED_BLOCKS_KEY); - }, - BackendWriteOp::WriteBlocksAtHeight(h, blocks) => { - tx.put_vec( - self.config.col_approval_data, - &blocks_at_height_key(h), - blocks.encode(), - ); - }, - BackendWriteOp::DeleteBlocksAtHeight(h) => { - tx.delete(self.config.col_approval_data, &blocks_at_height_key(h)); - }, - BackendWriteOp::WriteBlockEntry(block_entry) => { - let block_entry: BlockEntry = block_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &block_entry_key(&block_entry.block_hash), - block_entry.encode(), - ); - }, - BackendWriteOp::DeleteBlockEntry(hash) => { - tx.delete(self.config.col_approval_data, &block_entry_key(&hash)); - }, - BackendWriteOp::WriteCandidateEntry(candidate_entry) => { - let candidate_entry: CandidateEntry = candidate_entry.into(); - tx.put_vec( - self.config.col_approval_data, - &candidate_entry_key(&candidate_entry.candidate.hash()), - candidate_entry.encode(), - ); - }, - BackendWriteOp::DeleteCandidateEntry(candidate_hash) => { - tx.delete(self.config.col_approval_data, &candidate_entry_key(&candidate_hash)); - }, - } - } - - self.inner.write(tx).map_err(|e| e.into()) - } -} - -/// A range from earliest..last block number stored within the DB. -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct StoredBlockRange(pub BlockNumber, pub BlockNumber); - // slot_duration * 2 + DelayTranche gives the number of delay tranches since the // unix epoch. #[derive(Encode, Decode, Clone, Copy, Debug, PartialEq)] @@ -172,13 +46,6 @@ pub struct Tick(u64); /// Convenience type definition pub type Bitfield = BitVec; -/// The database config. -#[derive(Debug, Clone, Copy)] -pub struct Config { - /// The column family in the database where data is stored. - pub col_approval_data: u32, -} - /// Details pertaining to our assignment on a block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] pub struct OurAssignment { @@ -201,15 +68,6 @@ pub struct TrancheEntry { pub assignments: Vec<(ValidatorIndex, Tick)>, } -/// Metadata about our approval signature -#[derive(Encode, Decode, Debug, Clone, PartialEq)] -pub struct OurApproval { - /// The signature for the candidates hashes pointed by indices. - pub signature: ValidatorSignature, - /// The indices of the candidates signed in this approval. - pub signed_candidates_indices: CandidateBitfield, -} - /// Metadata regarding approval of a particular candidate within the context of some /// particular block. #[derive(Encode, Decode, Debug, Clone, PartialEq)] @@ -217,7 +75,7 @@ pub struct ApprovalEntry { pub tranches: Vec, pub backing_group: GroupIndex, pub our_assignment: Option, - pub our_approval_sig: Option, + pub our_approval_sig: Option, // `n_validators` bits. pub assigned_validators: Bitfield, pub approved: bool, @@ -254,25 +112,12 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, - // A list of candidates that has been approved, but we didn't not sign and - // advertise the vote yet. - pub candidates_pending_signature: BTreeMap, // Assignments we already distributed. A 1 bit means the candidate index for which // we already have sent out an assignment. We need this to avoid distributing // multiple core assignments more than once. pub distributed_assignments: Bitfield, } -#[derive(Encode, Decode, Debug, Clone, PartialEq)] - -/// Context needed for creating an approval signature for a given candidate. -pub struct CandidateSigningContext { - /// The candidate hash, to be included in the signature. - pub candidate_hash: CandidateHash, - /// The latest tick we have to create and send the approval. - pub sign_no_later_than_tick: Tick, -} - impl From for Tick { fn from(tick: crate::Tick) -> Tick { Tick(tick) @@ -285,118 +130,6 @@ impl From for crate::Tick { } } -/// Errors while accessing things from the DB. -#[derive(Debug, derive_more::From, derive_more::Display)] -pub enum Error { - Io(std::io::Error), - InvalidDecoding(parity_scale_codec::Error), - InternalError(SubsystemError), -} - -impl std::error::Error for Error {} - -/// Result alias for DB errors. -pub type Result = std::result::Result; - -pub(crate) fn load_decode( - store: &dyn Database, - col_approval_data: u32, - key: &[u8], -) -> Result> { - match store.get(col_approval_data, key)? { - None => Ok(None), - Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into), - } -} - -/// The key a given block entry is stored under. -pub(crate) fn block_entry_key(block_hash: &Hash) -> [u8; 46] { - const BLOCK_ENTRY_PREFIX: [u8; 14] = *b"Approvals_blck"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&BLOCK_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(block_hash.as_ref()); - - key -} - -/// The key a given candidate entry is stored under. -pub(crate) fn candidate_entry_key(candidate_hash: &CandidateHash) -> [u8; 46] { - const CANDIDATE_ENTRY_PREFIX: [u8; 14] = *b"Approvals_cand"; - - let mut key = [0u8; 14 + 32]; - key[0..14].copy_from_slice(&CANDIDATE_ENTRY_PREFIX); - key[14..][..32].copy_from_slice(candidate_hash.0.as_ref()); - - key -} - -/// The key a set of block hashes corresponding to a block number is stored under. -pub(crate) fn blocks_at_height_key(block_number: BlockNumber) -> [u8; 16] { - const BLOCKS_AT_HEIGHT_PREFIX: [u8; 12] = *b"Approvals_at"; - - let mut key = [0u8; 12 + 4]; - key[0..12].copy_from_slice(&BLOCKS_AT_HEIGHT_PREFIX); - block_number.using_encoded(|s| key[12..16].copy_from_slice(s)); - - key -} - -/// Return all blocks which have entries in the DB, ascending, by height. -pub fn load_all_blocks(store: &dyn Database, config: &Config) -> SubsystemResult> { - let mut hashes = Vec::new(); - if let Some(stored_blocks) = load_stored_blocks(store, config)? { - for height in stored_blocks.0..stored_blocks.1 { - let blocks = load_blocks_at_height(store, config, &height)?; - hashes.extend(blocks); - } - } - - Ok(hashes) -} - -/// Load the stored-blocks key from the state. -pub fn load_stored_blocks( - store: &dyn Database, - config: &Config, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, STORED_BLOCKS_KEY) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a blocks-at-height entry for a given block number. -pub fn load_blocks_at_height( - store: &dyn Database, - config: &Config, - block_number: &BlockNumber, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &blocks_at_height_key(*block_number)) - .map(|x| x.unwrap_or_default()) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a block entry from the aux store. -pub fn load_block_entry( - store: &dyn Database, - config: &Config, - block_hash: &Hash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - -/// Load a candidate entry from the aux store in current version format. -pub fn load_candidate_entry( - store: &dyn Database, - config: &Config, - candidate_hash: &CandidateHash, -) -> SubsystemResult> { - load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) - .map(|u: Option| u.map(|v| v.into())) - .map_err(|e| SubsystemError::with_origin("approval-voting", e)) -} - /// Load a candidate entry from the aux store in v1 format. pub fn load_candidate_entry_v1( store: &dyn Database, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs index 31b66d88577f..1121660fe658 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -16,13 +16,23 @@ //! Tests for the aux-schema of approval voting. -use super::{DbBackend, StoredBlockRange, *}; use crate::{ + approval_db::{ + common::{DbBackend, StoredBlockRange, *}, + v2::*, + v3::{load_block_entry_v2, load_candidate_entry_v2}, + }, backend::{Backend, OverlayedBackend}, ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo}, }; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, +}; + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::Id as ParaId; +use sp_consensus_slots::Slot; use std::{collections::HashMap, sync::Arc}; use ::test_helpers::{dummy_candidate_receipt, dummy_candidate_receipt_bad_sig, dummy_hash}; @@ -56,7 +66,6 @@ fn make_block_entry( approved_bitfield: make_bitvec(candidates.len()), candidates, children: Vec::new(), - candidates_pending_signature: Default::default(), distributed_assignments: Default::default(), } } @@ -111,7 +120,10 @@ fn read_write() { overlay_db.write_stored_block_range(range.clone()); overlay_db.write_blocks_at_height(1, at_height.clone()); overlay_db.write_block_entry(block_entry.clone().into()); - overlay_db.write_candidate_entry(candidate_entry.clone().into()); + overlay_db.write_candidate_entry(crate::persisted_entries::CandidateEntry::from_v2( + candidate_entry.clone(), + 0, + )); let write_ops = overlay_db.into_write_ops(); db.write(write_ops).unwrap(); @@ -119,11 +131,11 @@ fn read_write() { assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range)); assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(), Some(block_entry.into()) ); assert_eq!( - load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(), + load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(), Some(candidate_entry.into()), ); @@ -135,8 +147,8 @@ fn read_write() { db.write(write_ops).unwrap(); assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty()); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none()); - assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none()); + assert!(load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash) .unwrap() .is_none()); } @@ -197,25 +209,27 @@ fn add_block_entry_works() { db.write(write_ops).unwrap(); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), Some(block_entry_a.into()) ); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), Some(block_entry_b.into()) ); - let candidate_entry_a = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_a) - .unwrap() - .unwrap(); + let candidate_entry_a = + load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash_a) + .unwrap() + .unwrap(); assert_eq!( candidate_entry_a.block_assignments.keys().collect::>(), vec![&block_hash_a, &block_hash_b] ); - let candidate_entry_b = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_b) - .unwrap() - .unwrap(); + let candidate_entry_b = + load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &candidate_hash_b) + .unwrap() + .unwrap(); assert_eq!(candidate_entry_b.block_assignments.keys().collect::>(), vec![&block_hash_b]); } @@ -244,11 +258,11 @@ fn add_block_entry_adds_child() { block_entry_a.children.push(block_hash_b); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), Some(block_entry_a.into()) ); assert_eq!( - load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), Some(block_entry_b.into()) ); } @@ -366,13 +380,15 @@ fn canonicalize_works() { for (c_hash, in_blocks) in expected { let (entry, in_blocks) = match in_blocks { None => { - assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash) + assert!(load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &c_hash) .unwrap() .is_none()); continue }, Some(i) => ( - load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().unwrap(), + load_candidate_entry_v2(store.as_ref(), &TEST_CONFIG, &c_hash) + .unwrap() + .unwrap(), i, ), }; @@ -389,13 +405,13 @@ fn canonicalize_works() { for (hash, with_candidates) in expected { let (entry, with_candidates) = match with_candidates { None => { - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash) .unwrap() .is_none()); continue }, Some(i) => - (load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i), + (load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i), }; assert_eq!(entry.candidates.len(), with_candidates.len()); @@ -511,22 +527,22 @@ fn force_approve_works() { let write_ops = overlay_db.into_write_ops(); db.write(write_ops).unwrap(); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a,) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_a,) .unwrap() .unwrap() .approved_bitfield .all()); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b,) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_b,) .unwrap() .unwrap() .approved_bitfield .all()); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_c,) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_c,) .unwrap() .unwrap() .approved_bitfield .not_any()); - assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_d,) + assert!(load_block_entry_v2(store.as_ref(), &TEST_CONFIG, &block_hash_d,) .unwrap() .unwrap() .approved_bitfield diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs new file mode 100644 index 000000000000..3ffb66015215 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs @@ -0,0 +1,237 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Approval DB migration helpers. +use super::*; +use crate::{ + approval_db::common::{ + block_entry_key, candidate_entry_key, + migration_helpers::{dummy_assignment_cert, make_bitvec}, + Config, Error, Result, StoredBlockRange, + }, + backend::{Backend, V2ReadBackend}, +}; +use polkadot_node_primitives::approval::v1::AssignmentCertKind; +use polkadot_node_subsystem_util::database::Database; +use sp_application_crypto::sp_core::H256; +use std::{collections::HashSet, sync::Arc}; + +/// Migrates `BlockEntry`, `CandidateEntry`, `ApprovalEntry` and `OurApproval` to version 3. +/// Returns on any error. +/// Must only be used in parachains DB migration code - `polkadot-service` crate. +pub fn v2_to_latest(db: Arc, config: Config) -> Result<()> { + let mut backend = crate::DbBackend::new(db, config); + let all_blocks = backend + .load_all_blocks() + .map_err(|e| Error::InternalError(e))? + .iter() + .filter_map(|block_hash| { + backend + .load_block_entry_v2(block_hash) + .map_err(|e| Error::InternalError(e)) + .ok()? + }) + .collect::>(); + + gum::info!( + target: crate::LOG_TARGET, + "Migrating candidate entries on top of {} blocks", + all_blocks.len() + ); + + let mut overlay = crate::OverlayedBackend::new(&backend); + let mut counter = 0; + // Get all candidate entries, approval entries and convert each of them. + for block in all_blocks { + for (candidate_index, (_core_index, candidate_hash)) in + block.candidates().iter().enumerate() + { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend + .load_candidate_entry_v2(&candidate_hash, candidate_index as CandidateIndex) + .map_err(|e| Error::InternalError(e))? + { + // Write the updated representation. + overlay.write_candidate_entry(candidate_entry); + counter += 1; + } + } + overlay.write_block_entry(block); + } + + gum::info!(target: crate::LOG_TARGET, "Migrated {} entries", counter); + + // Commit all changes to DB. + let write_ops = overlay.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(()) +} + +// Checks if the migration doesn't leave the DB in an unsane state. +// This function is to be used in tests. +pub fn v1_to_latest_sanity_check( + db: Arc, + config: Config, + expected_candidates: HashSet, +) -> Result<()> { + let backend = crate::DbBackend::new(db, config); + + let all_blocks = backend + .load_all_blocks() + .unwrap() + .iter() + .map(|block_hash| backend.load_block_entry(block_hash).unwrap().unwrap()) + .collect::>(); + + let mut candidates = HashSet::new(); + + // Iterate all blocks and approval entries. + for block in all_blocks { + for (_core_index, candidate_hash) in block.candidates() { + // Loading the candidate will also perform the conversion to the updated format and + // return that represantation. + if let Some(candidate_entry) = backend.load_candidate_entry(&candidate_hash).unwrap() { + candidates.insert(candidate_entry.candidate.hash()); + } + } + } + + assert_eq!(candidates, expected_candidates); + + Ok(()) +} + +// Fills the db with dummy data in v1 scheme. +pub fn v2_fill_test_data( + db: Arc, + config: Config, + dummy_candidate_create: F, +) -> Result> +where + F: Fn(H256) -> CandidateReceipt, +{ + let mut backend = crate::DbBackend::new(db.clone(), config); + let mut overlay_db = crate::OverlayedBackend::new(&backend); + let mut expected_candidates = HashSet::new(); + + const RELAY_BLOCK_COUNT: u32 = 10; + + let range = StoredBlockRange(1, 11); + overlay_db.write_stored_block_range(range.clone()); + + for relay_number in 1..=RELAY_BLOCK_COUNT { + let relay_hash = Hash::repeat_byte(relay_number as u8); + let assignment_core_index = CoreIndex(relay_number); + let candidate = dummy_candidate_create(relay_hash); + let candidate_hash = candidate.hash(); + + let at_height = vec![relay_hash]; + + let block_entry = make_block_entry_v2( + relay_hash, + Default::default(), + relay_number, + vec![(assignment_core_index, candidate_hash)], + ); + + let dummy_assignment = crate::approval_db::v2::OurAssignment { + cert: dummy_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }).into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + }; + + let candidate_entry = crate::approval_db::v2::CandidateEntry { + candidate, + session: 123, + block_assignments: vec![( + relay_hash, + crate::approval_db::v2::ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: Some(dummy_assignment), + our_approval_sig: None, + approved: false, + assigned_validators: make_bitvec(1), + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + overlay_db.write_blocks_at_height(relay_number, at_height.clone()); + expected_candidates.insert(candidate_entry.candidate.hash()); + + db.write(write_candidate_entry_v2(candidate_entry, config)).unwrap(); + db.write(write_block_entry_v2(block_entry, config)).unwrap(); + } + + let write_ops = overlay_db.into_write_ops(); + backend.write(write_ops).unwrap(); + + Ok(expected_candidates) +} + +fn make_block_entry_v2( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> crate::approval_db::v2::BlockEntry { + crate::approval_db::v2::BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + distributed_assignments: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + } +} + +// Low level DB helper to write a candidate entry in v1 scheme. +fn write_candidate_entry_v2( + candidate_entry: crate::approval_db::v2::CandidateEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &candidate_entry_key(&candidate_entry.candidate.hash()), + candidate_entry.encode(), + ); + tx +} + +// Low level DB helper to write a block entry in v1 scheme. +fn write_block_entry_v2( + block_entry: crate::approval_db::v2::BlockEntry, + config: Config, +) -> DBTransaction { + let mut tx = DBTransaction::new(); + tx.put_vec( + config.col_approval_data, + &block_entry_key(&block_entry.block_hash), + block_entry.encode(), + ); + tx +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs new file mode 100644 index 000000000000..4483a4740a1f --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs @@ -0,0 +1,139 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Version 3 of the DB schema. +//! +//! Version 3 modifies the our_approval format of `ApprovalEntry` +//! and adds a new field `pending_signatures` for `BlockEntry` +//! and adds a new field `pending_signatures` for `BlockEntry` + +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_primitives::approval::v2::CandidateBitfield; +use polkadot_node_subsystem::SubsystemResult; +use polkadot_node_subsystem_util::database::{DBTransaction, Database}; +use polkadot_overseer::SubsystemError; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, GroupIndex, Hash, + SessionIndex, ValidatorIndex, ValidatorSignature, +}; + +use sp_consensus_slots::Slot; + +use std::collections::BTreeMap; + +use super::common::{block_entry_key, candidate_entry_key, load_decode, Config}; + +/// Re-export this structs as v3 since they did not change between v2 and v3. +pub use super::v2::{Bitfield, OurAssignment, Tick, TrancheEntry}; + +pub mod migration_helpers; + +#[cfg(test)] +pub mod tests; + +/// Metadata about our approval signature +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct OurApproval { + /// The signature for the candidates hashes pointed by indices. + pub signature: ValidatorSignature, + /// The indices of the candidates signed in this approval. + pub signed_candidates_indices: CandidateBitfield, +} + +/// Metadata regarding approval of a particular candidate within the context of some +/// particular block. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct ApprovalEntry { + pub tranches: Vec, + pub backing_group: GroupIndex, + pub our_assignment: Option, + pub our_approval_sig: Option, + // `n_validators` bits. + pub assigned_validators: Bitfield, + pub approved: bool, +} + +/// Metadata regarding approval of a particular candidate. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct CandidateEntry { + pub candidate: CandidateReceipt, + pub session: SessionIndex, + // Assignments are based on blocks, so we need to track assignments separately + // based on the block we are looking at. + pub block_assignments: BTreeMap, + pub approvals: Bitfield, +} + +/// Metadata regarding approval of a particular block, by way of approval of the +/// candidates contained within it. +#[derive(Encode, Decode, Debug, Clone, PartialEq)] +pub struct BlockEntry { + pub block_hash: Hash, + pub block_number: BlockNumber, + pub parent_hash: Hash, + pub session: SessionIndex, + pub slot: Slot, + /// Random bytes derived from the VRF submitted within the block by the block + /// author as a credential and used as input to approval assignment criteria. + pub relay_vrf_story: [u8; 32], + // The candidates included as-of this block and the index of the core they are + // leaving. Sorted ascending by core index. + pub candidates: Vec<(CoreIndex, CandidateHash)>, + // A bitfield where the i'th bit corresponds to the i'th candidate in `candidates`. + // The i'th bit is `true` iff the candidate has been approved in the context of this + // block. The block can be considered approved if the bitfield has all bits set to `true`. + pub approved_bitfield: Bitfield, + pub children: Vec, + // A list of candidates that has been approved, but we didn't not sign and + // advertise the vote yet. + pub candidates_pending_signature: BTreeMap, + // Assignments we already distributed. A 1 bit means the candidate index for which + // we already have sent out an assignment. We need this to avoid distributing + // multiple core assignments more than once. + pub distributed_assignments: Bitfield, +} + +#[derive(Encode, Decode, Debug, Clone, PartialEq)] + +/// Context needed for creating an approval signature for a given candidate. +pub struct CandidateSigningContext { + /// The candidate hash, to be included in the signature. + pub candidate_hash: CandidateHash, + /// The latest tick we have to create and send the approval. + pub sign_no_later_than_tick: Tick, +} + +/// Load a candidate entry from the aux store in v2 format. +pub fn load_candidate_entry_v2( + store: &dyn Database, + config: &Config, + candidate_hash: &CandidateHash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &candidate_entry_key(candidate_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} + +/// Load a block entry from the aux store in v2 format. +pub fn load_block_entry_v2( + store: &dyn Database, + config: &Config, + block_hash: &Hash, +) -> SubsystemResult> { + load_decode(store, config.col_approval_data, &block_entry_key(block_hash)) + .map(|u: Option| u.map(|v| v.into())) + .map_err(|e| SubsystemError::with_origin("approval-voting", e)) +} diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs new file mode 100644 index 000000000000..7ff6433b5365 --- /dev/null +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs @@ -0,0 +1,580 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Tests for the aux-schema of approval voting. + +use crate::{ + approval_db::{ + common::{DbBackend, StoredBlockRange, *}, + v3::*, + }, + backend::{Backend, OverlayedBackend}, + ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo}, +}; +use polkadot_primitives::{ + BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, +}; + +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; +use polkadot_node_subsystem_util::database::Database; +use polkadot_primitives::Id as ParaId; +use sp_consensus_slots::Slot; +use std::{collections::HashMap, sync::Arc}; + +use ::test_helpers::{dummy_candidate_receipt, dummy_candidate_receipt_bad_sig, dummy_hash}; + +const DATA_COL: u32 = 0; + +const NUM_COLUMNS: u32 = 1; + +const TEST_CONFIG: Config = Config { col_approval_data: DATA_COL }; + +fn make_db() -> (DbBackend, Arc) { + let db = kvdb_memorydb::create(NUM_COLUMNS); + let db = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); + let db_writer: Arc = Arc::new(db); + (DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer) +} + +fn make_block_entry( + block_hash: Hash, + parent_hash: Hash, + block_number: BlockNumber, + candidates: Vec<(CoreIndex, CandidateHash)>, +) -> BlockEntry { + BlockEntry { + block_hash, + parent_hash, + block_number, + session: 1, + slot: Slot::from(1), + relay_vrf_story: [0u8; 32], + approved_bitfield: make_bitvec(candidates.len()), + candidates, + children: Vec::new(), + candidates_pending_signature: Default::default(), + distributed_assignments: Default::default(), + } +} + +fn make_bitvec(len: usize) -> BitVec { + bitvec::bitvec![u8, BitOrderLsb0; 0; len] +} + +fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { + let mut c = dummy_candidate_receipt(dummy_hash()); + + c.descriptor.para_id = para_id; + c.descriptor.relay_parent = relay_parent; + + c +} + +#[test] +fn read_write() { + let (mut db, store) = make_db(); + + let hash_a = Hash::repeat_byte(1); + let hash_b = Hash::repeat_byte(2); + let candidate_hash = dummy_candidate_receipt_bad_sig(dummy_hash(), None).hash(); + + let range = StoredBlockRange(10, 20); + let at_height = vec![hash_a, hash_b]; + + let block_entry = + make_block_entry(hash_a, Default::default(), 1, vec![(CoreIndex(0), candidate_hash)]); + + let candidate_entry = CandidateEntry { + candidate: dummy_candidate_receipt_bad_sig(dummy_hash(), None), + session: 5, + block_assignments: vec![( + hash_a, + ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(1), + our_assignment: None, + our_approval_sig: None, + assigned_validators: Default::default(), + approved: false, + }, + )] + .into_iter() + .collect(), + approvals: Default::default(), + }; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(range.clone()); + overlay_db.write_blocks_at_height(1, at_height.clone()); + overlay_db.write_block_entry(block_entry.clone().into()); + overlay_db.write_candidate_entry(candidate_entry.clone().into()); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range)); + assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(), + Some(block_entry.into()) + ); + assert_eq!( + load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(), + Some(candidate_entry.into()), + ); + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.delete_blocks_at_height(1); + overlay_db.delete_block_entry(&hash_a); + overlay_db.delete_candidate_entry(&candidate_hash); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none()); + assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash) + .unwrap() + .is_none()); +} + +#[test] +fn add_block_entry_works() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + + let candidate_receipt_a = make_candidate(ParaId::from(1_u32), parent_hash); + let candidate_receipt_b = make_candidate(ParaId::from(2_u32), parent_hash); + + let candidate_hash_a = candidate_receipt_a.hash(); + let candidate_hash_b = candidate_receipt_b.hash(); + + let block_number = 10; + + let block_entry_a = make_block_entry( + block_hash_a, + parent_hash, + block_number, + vec![(CoreIndex(0), candidate_hash_a)], + ); + + let block_entry_b = make_block_entry( + block_hash_b, + parent_hash, + block_number, + vec![(CoreIndex(0), candidate_hash_a), (CoreIndex(1), candidate_hash_b)], + ); + + let n_validators = 10; + + let mut new_candidate_info = HashMap::new(); + new_candidate_info + .insert(candidate_hash_a, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(0), None)); + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |h| { + new_candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + new_candidate_info + .insert(candidate_hash_b, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(1), None)); + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |h| { + new_candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + Some(block_entry_a.into()) + ); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + Some(block_entry_b.into()) + ); + + let candidate_entry_a = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_a) + .unwrap() + .unwrap(); + assert_eq!( + candidate_entry_a.block_assignments.keys().collect::>(), + vec![&block_hash_a, &block_hash_b] + ); + + let candidate_entry_b = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_b) + .unwrap() + .unwrap(); + assert_eq!(candidate_entry_b.block_assignments.keys().collect::>(), vec![&block_hash_b]); +} + +#[test] +fn add_block_entry_adds_child() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + + let mut block_entry_a = make_block_entry(block_hash_a, parent_hash, 1, Vec::new()); + + let block_entry_b = make_block_entry(block_hash_b, block_hash_a, 2, Vec::new()); + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap(); + + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap(); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + block_entry_a.children.push(block_hash_b); + + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), + Some(block_entry_a.into()) + ); + assert_eq!( + load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), + Some(block_entry_b.into()) + ); +} + +#[test] +fn canonicalize_works() { + let (mut db, store) = make_db(); + + // -> B1 -> C1 -> D1 + // A -> B2 -> C2 -> D2 + // + // We'll canonicalize C1. Everytning except D1 should disappear. + // + // Candidates: + // Cand1 in B2 + // Cand2 in C2 + // Cand3 in C2 and D1 + // Cand4 in D1 + // Cand5 in D2 + // Only Cand3 and Cand4 should remain after canonicalize. + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(StoredBlockRange(1, 5)); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let genesis = Hash::repeat_byte(0); + + let block_hash_a = Hash::repeat_byte(1); + let block_hash_b1 = Hash::repeat_byte(2); + let block_hash_b2 = Hash::repeat_byte(3); + let block_hash_c1 = Hash::repeat_byte(4); + let block_hash_c2 = Hash::repeat_byte(5); + let block_hash_d1 = Hash::repeat_byte(6); + let block_hash_d2 = Hash::repeat_byte(7); + + let candidate_receipt_genesis = make_candidate(ParaId::from(1_u32), genesis); + let candidate_receipt_a = make_candidate(ParaId::from(2_u32), block_hash_a); + let candidate_receipt_b = make_candidate(ParaId::from(3_u32), block_hash_a); + let candidate_receipt_b1 = make_candidate(ParaId::from(4_u32), block_hash_b1); + let candidate_receipt_c1 = make_candidate(ParaId::from(5_u32), block_hash_c1); + + let cand_hash_1 = candidate_receipt_genesis.hash(); + let cand_hash_2 = candidate_receipt_a.hash(); + let cand_hash_3 = candidate_receipt_b.hash(); + let cand_hash_4 = candidate_receipt_b1.hash(); + let cand_hash_5 = candidate_receipt_c1.hash(); + + let block_entry_a = make_block_entry(block_hash_a, genesis, 1, Vec::new()); + let block_entry_b1 = make_block_entry(block_hash_b1, block_hash_a, 2, Vec::new()); + let block_entry_b2 = + make_block_entry(block_hash_b2, block_hash_a, 2, vec![(CoreIndex(0), cand_hash_1)]); + let block_entry_c1 = make_block_entry(block_hash_c1, block_hash_b1, 3, Vec::new()); + let block_entry_c2 = make_block_entry( + block_hash_c2, + block_hash_b2, + 3, + vec![(CoreIndex(0), cand_hash_2), (CoreIndex(1), cand_hash_3)], + ); + let block_entry_d1 = make_block_entry( + block_hash_d1, + block_hash_c1, + 4, + vec![(CoreIndex(0), cand_hash_3), (CoreIndex(1), cand_hash_4)], + ); + let block_entry_d2 = + make_block_entry(block_hash_d2, block_hash_c2, 4, vec![(CoreIndex(0), cand_hash_5)]); + + let candidate_info = { + let mut candidate_info = HashMap::new(); + candidate_info.insert( + cand_hash_1, + NewCandidateInfo::new(candidate_receipt_genesis, GroupIndex(1), None), + ); + + candidate_info + .insert(cand_hash_2, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(2), None)); + + candidate_info + .insert(cand_hash_3, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(3), None)); + + candidate_info + .insert(cand_hash_4, NewCandidateInfo::new(candidate_receipt_b1, GroupIndex(4), None)); + + candidate_info + .insert(cand_hash_5, NewCandidateInfo::new(candidate_receipt_c1, GroupIndex(5), None)); + + candidate_info + }; + + // now insert all the blocks. + let blocks = vec![ + block_entry_a.clone(), + block_entry_b1.clone(), + block_entry_b2.clone(), + block_entry_c1.clone(), + block_entry_c2.clone(), + block_entry_d1.clone(), + block_entry_d2.clone(), + ]; + + let mut overlay_db = OverlayedBackend::new(&db); + for block_entry in blocks { + add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| { + candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + } + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let check_candidates_in_store = |expected: Vec<(CandidateHash, Option>)>| { + for (c_hash, in_blocks) in expected { + let (entry, in_blocks) = match in_blocks { + None => { + assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash) + .unwrap() + .is_none()); + continue + }, + Some(i) => ( + load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().unwrap(), + i, + ), + }; + + assert_eq!(entry.block_assignments.len(), in_blocks.len()); + + for x in in_blocks { + assert!(entry.block_assignments.contains_key(&x)); + } + } + }; + + let check_blocks_in_store = |expected: Vec<(Hash, Option>)>| { + for (hash, with_candidates) in expected { + let (entry, with_candidates) = match with_candidates { + None => { + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash) + .unwrap() + .is_none()); + continue + }, + Some(i) => + (load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i), + }; + + assert_eq!(entry.candidates.len(), with_candidates.len()); + + for x in with_candidates { + assert!(entry.candidates.iter().any(|(_, c)| c == &x)); + } + } + }; + + check_candidates_in_store(vec![ + (cand_hash_1, Some(vec![block_hash_b2])), + (cand_hash_2, Some(vec![block_hash_c2])), + (cand_hash_3, Some(vec![block_hash_c2, block_hash_d1])), + (cand_hash_4, Some(vec![block_hash_d1])), + (cand_hash_5, Some(vec![block_hash_d2])), + ]); + + check_blocks_in_store(vec![ + (block_hash_a, Some(vec![])), + (block_hash_b1, Some(vec![])), + (block_hash_b2, Some(vec![cand_hash_1])), + (block_hash_c1, Some(vec![])), + (block_hash_c2, Some(vec![cand_hash_2, cand_hash_3])), + (block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])), + (block_hash_d2, Some(vec![cand_hash_5])), + ]); + + let mut overlay_db = OverlayedBackend::new(&db); + canonicalize(&mut overlay_db, 3, block_hash_c1).unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(), + StoredBlockRange(4, 5) + ); + + check_candidates_in_store(vec![ + (cand_hash_1, None), + (cand_hash_2, None), + (cand_hash_3, Some(vec![block_hash_d1])), + (cand_hash_4, Some(vec![block_hash_d1])), + (cand_hash_5, None), + ]); + + check_blocks_in_store(vec![ + (block_hash_a, None), + (block_hash_b1, None), + (block_hash_b2, None), + (block_hash_c1, None), + (block_hash_c2, None), + (block_hash_d1, Some(vec![cand_hash_3, cand_hash_4])), + (block_hash_d2, None), + ]); +} + +#[test] +fn force_approve_works() { + let (mut db, store) = make_db(); + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + overlay_db.write_stored_block_range(StoredBlockRange(1, 4)); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + let candidate_hash = CandidateHash(Hash::repeat_byte(42)); + let single_candidate_vec = vec![(CoreIndex(0), candidate_hash)]; + let candidate_info = { + let mut candidate_info = HashMap::new(); + candidate_info.insert( + candidate_hash, + NewCandidateInfo::new( + make_candidate(ParaId::from(1_u32), Default::default()), + GroupIndex(1), + None, + ), + ); + + candidate_info + }; + + let block_hash_a = Hash::repeat_byte(1); // 1 + let block_hash_b = Hash::repeat_byte(2); + let block_hash_c = Hash::repeat_byte(3); + let block_hash_d = Hash::repeat_byte(4); // 4 + + let block_entry_a = + make_block_entry(block_hash_a, Default::default(), 1, single_candidate_vec.clone()); + let block_entry_b = + make_block_entry(block_hash_b, block_hash_a, 2, single_candidate_vec.clone()); + let block_entry_c = + make_block_entry(block_hash_c, block_hash_b, 3, single_candidate_vec.clone()); + let block_entry_d = + make_block_entry(block_hash_d, block_hash_c, 4, single_candidate_vec.clone()); + + let blocks = vec![ + block_entry_a.clone(), + block_entry_b.clone(), + block_entry_c.clone(), + block_entry_d.clone(), + ]; + + let mut overlay_db = OverlayedBackend::new(&db); + for block_entry in blocks { + add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| { + candidate_info.get(h).map(|x| x.clone()) + }) + .unwrap(); + } + let approved_hashes = force_approve(&mut overlay_db, block_hash_d, 2).unwrap(); + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a,) + .unwrap() + .unwrap() + .approved_bitfield + .all()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b,) + .unwrap() + .unwrap() + .approved_bitfield + .all()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_c,) + .unwrap() + .unwrap() + .approved_bitfield + .not_any()); + assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_d,) + .unwrap() + .unwrap() + .approved_bitfield + .not_any()); + assert_eq!(approved_hashes, vec![block_hash_b, block_hash_a]); +} + +#[test] +fn load_all_blocks_works() { + let (mut db, store) = make_db(); + + let parent_hash = Hash::repeat_byte(1); + let block_hash_a = Hash::repeat_byte(2); + let block_hash_b = Hash::repeat_byte(69); + let block_hash_c = Hash::repeat_byte(42); + + let block_number = 10; + + let block_entry_a = make_block_entry(block_hash_a, parent_hash, block_number, vec![]); + + let block_entry_b = make_block_entry(block_hash_b, parent_hash, block_number, vec![]); + + let block_entry_c = make_block_entry(block_hash_c, block_hash_a, block_number + 1, vec![]); + + let n_validators = 10; + + let mut overlay_db = OverlayedBackend::new(&db); + add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap(); + + // add C before B to test sorting. + add_block_entry(&mut overlay_db, block_entry_c.clone().into(), n_validators, |_| None).unwrap(); + + add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap(); + + let write_ops = overlay_db.into_write_ops(); + db.write(write_ops).unwrap(); + + assert_eq!( + load_all_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), + vec![block_hash_a, block_hash_b, block_hash_c], + ) +} diff --git a/polkadot/node/core/approval-voting/src/backend.rs b/polkadot/node/core/approval-voting/src/backend.rs index 801120b210b5..9ce25334c0fa 100644 --- a/polkadot/node/core/approval-voting/src/backend.rs +++ b/polkadot/node/core/approval-voting/src/backend.rs @@ -27,7 +27,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, CandidateIndex, Hash}; use std::collections::HashMap; use super::{ - approval_db::v2::StoredBlockRange, + approval_db::common::StoredBlockRange, persisted_entries::{BlockEntry, CandidateEntry}, }; @@ -79,6 +79,19 @@ pub trait V1ReadBackend: Backend { fn load_block_entry_v1(&self, block_hash: &Hash) -> SubsystemResult>; } +/// A read only backend to enable db migration from version 2 of DB. +pub trait V2ReadBackend: Backend { + /// Load a candidate entry from the DB with scheme version 1. + fn load_candidate_entry_v2( + &self, + candidate_hash: &CandidateHash, + candidate_index: CandidateIndex, + ) -> SubsystemResult>; + + /// Load a block entry from the DB with scheme version 1. + fn load_block_entry_v2(&self, block_hash: &Hash) -> SubsystemResult>; +} + // Status of block range in the `OverlayedBackend`. #[derive(PartialEq)] enum BlockRangeStatus { diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index 47587eb27ef0..ba894c2a30ff 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -56,7 +56,7 @@ use futures::{channel::oneshot, prelude::*}; use std::collections::HashMap; -use super::approval_db::v2; +use super::approval_db::v3; use crate::{ backend::{Backend, OverlayedBackend}, criteria::{AssignmentCriteria, OurAssignment}, @@ -500,7 +500,7 @@ pub(crate) async fn handle_new_head( ctx.send_message(ChainSelectionMessage::Approved(block_hash)).await; } - let block_entry = v2::BlockEntry { + let block_entry = v3::BlockEntry { block_hash, parent_hash: block_header.parent_hash, block_number: block_header.number, @@ -593,7 +593,10 @@ pub(crate) async fn handle_new_head( #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::{approval_db::v2::DbBackend, RuntimeInfo, RuntimeInfoConfig}; + use crate::{ + approval_db::common::{load_block_entry, DbBackend}, + RuntimeInfo, RuntimeInfoConfig, + }; use ::test_helpers::{dummy_candidate_receipt, dummy_hash}; use assert_matches::assert_matches; use polkadot_node_primitives::{ @@ -615,7 +618,7 @@ pub(crate) mod tests { pub(crate) use sp_runtime::{Digest, DigestItem}; use std::{pin::Pin, sync::Arc}; - use crate::{approval_db::v2::Config as DatabaseConfig, criteria, BlockEntry}; + use crate::{approval_db::common::Config as DatabaseConfig, criteria, BlockEntry}; const DATA_COL: u32 = 0; @@ -1303,7 +1306,7 @@ pub(crate) mod tests { let (state, mut session_info_provider) = single_session_state(); overlay_db.write_block_entry( - v2::BlockEntry { + v3::BlockEntry { block_hash: parent_hash, parent_hash: Default::default(), block_number: 4, @@ -1346,11 +1349,10 @@ pub(crate) mod tests { assert_eq!(candidates[1].1.approvals().len(), 6); // the first candidate should be insta-approved // the second should not - let entry: BlockEntry = - v2::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) - .unwrap() - .unwrap() - .into(); + let entry: BlockEntry = load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash) + .unwrap() + .unwrap() + .into(); assert!(entry.is_candidate_approved(&candidates[0].0)); assert!(!entry.is_candidate_approved(&candidates[1].0)); }) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index e21ad331431d..c12ba99fc319 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -100,7 +100,7 @@ mod time; use crate::{ approval_checking::{Check, TranchesToApproveResult}, - approval_db::v2::{Config as DatabaseConfig, DbBackend}, + approval_db::common::{Config as DatabaseConfig, DbBackend}, backend::{Backend, OverlayedBackend}, criteria::InvalidAssignmentReason, persisted_entries::OurApproval, @@ -485,8 +485,8 @@ impl ApprovalVotingSubsystem { /// The operation is not allowed for blocks older than the last finalized one. pub fn revert_to(&self, hash: Hash) -> Result<(), SubsystemError> { let config = - approval_db::v2::Config { col_approval_data: self.db_config.col_approval_data }; - let mut backend = approval_db::v2::DbBackend::new(self.db.clone(), config); + approval_db::common::Config { col_approval_data: self.db_config.col_approval_data }; + let mut backend = approval_db::common::DbBackend::new(self.db.clone(), config); let mut overlay = OverlayedBackend::new(&backend); ops::revert_to(&mut overlay, hash)?; diff --git a/polkadot/node/core/approval-voting/src/ops.rs b/polkadot/node/core/approval-voting/src/ops.rs index a6f0ecf9d1f0..2a8fdba5aa36 100644 --- a/polkadot/node/core/approval-voting/src/ops.rs +++ b/polkadot/node/core/approval-voting/src/ops.rs @@ -25,7 +25,7 @@ use polkadot_primitives::{BlockNumber, CandidateHash, CandidateReceipt, GroupInd use std::collections::{hash_map::Entry, BTreeMap, HashMap}; use super::{ - approval_db::v2::{OurAssignment, StoredBlockRange}, + approval_db::{common::StoredBlockRange, v2::OurAssignment}, backend::{Backend, OverlayedBackend}, persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry}, LOG_TARGET, diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index d9906c45ecc4..05da6f844063 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -77,15 +77,15 @@ impl From for crate::approval_db::v2::TrancheEntry { } } -impl From for OurApproval { - fn from(approval: crate::approval_db::v2::OurApproval) -> Self { +impl From for OurApproval { + fn from(approval: crate::approval_db::v3::OurApproval) -> Self { Self { signature: approval.signature, signed_candidates_indices: approval.signed_candidates_indices, } } } -impl From for crate::approval_db::v2::OurApproval { +impl From for crate::approval_db::v3::OurApproval { fn from(approval: OurApproval) -> Self { Self { signature: approval.signature, @@ -105,10 +105,16 @@ pub struct OurApproval { impl OurApproval { /// Converts a ValidatorSignature to an OurApproval. - /// It used in converting the database from v1 to v2. + /// It used in converting the database from v1 to latest. pub fn from_v1(value: ValidatorSignature, candidate_index: CandidateIndex) -> Self { Self { signature: value, signed_candidates_indices: candidate_index.into() } } + + /// Converts a ValidatorSignature to an OurApproval. + /// It used in converting the database from v2 to latest. + pub fn from_v2(value: ValidatorSignature, candidate_index: CandidateIndex) -> Self { + Self::from_v1(value, candidate_index) + } } /// Metadata regarding approval of a particular candidate within the context of some /// particular block. @@ -267,7 +273,7 @@ impl ApprovalEntry { } } - // Convert an ApprovalEntry from v1 version to a v2 version + // Convert an ApprovalEntry from v1 version to latest version pub fn from_v1( value: crate::approval_db::v1::ApprovalEntry, candidate_index: CandidateIndex, @@ -283,10 +289,27 @@ impl ApprovalEntry { approved: value.approved, } } + + // Convert an ApprovalEntry from v1 version to latest version + pub fn from_v2( + value: crate::approval_db::v2::ApprovalEntry, + candidate_index: CandidateIndex, + ) -> Self { + ApprovalEntry { + tranches: value.tranches.into_iter().map(|tranche| tranche.into()).collect(), + backing_group: value.backing_group, + our_assignment: value.our_assignment.map(|assignment| assignment.into()), + our_approval_sig: value + .our_approval_sig + .map(|sig| OurApproval::from_v2(sig, candidate_index)), + assigned_validators: value.assigned_validators, + approved: value.approved, + } + } } -impl From for ApprovalEntry { - fn from(entry: crate::approval_db::v2::ApprovalEntry) -> Self { +impl From for ApprovalEntry { + fn from(entry: crate::approval_db::v3::ApprovalEntry) -> Self { ApprovalEntry { tranches: entry.tranches.into_iter().map(Into::into).collect(), backing_group: entry.backing_group, @@ -298,7 +321,7 @@ impl From for ApprovalEntry { } } -impl From for crate::approval_db::v2::ApprovalEntry { +impl From for crate::approval_db::v3::ApprovalEntry { fn from(entry: ApprovalEntry) -> Self { Self { tranches: entry.tranches.into_iter().map(Into::into).collect(), @@ -355,7 +378,7 @@ impl CandidateEntry { self.block_assignments.get(block_hash) } - /// Convert a CandidateEntry from a v1 to its v2 equivalent. + /// Convert a CandidateEntry from a v1 to its latest equivalent. pub fn from_v1( value: crate::approval_db::v1::CandidateEntry, candidate_index: CandidateIndex, @@ -371,10 +394,27 @@ impl CandidateEntry { session: value.session, } } + + /// Convert a CandidateEntry from a v2 to its latest equivalent. + pub fn from_v2( + value: crate::approval_db::v2::CandidateEntry, + candidate_index: CandidateIndex, + ) -> Self { + Self { + approvals: value.approvals, + block_assignments: value + .block_assignments + .into_iter() + .map(|(h, ae)| (h, ApprovalEntry::from_v2(ae, candidate_index))) + .collect(), + candidate: value.candidate, + session: value.session, + } + } } -impl From for CandidateEntry { - fn from(entry: crate::approval_db::v2::CandidateEntry) -> Self { +impl From for CandidateEntry { + fn from(entry: crate::approval_db::v3::CandidateEntry) -> Self { CandidateEntry { candidate: entry.candidate, session: entry.session, @@ -388,7 +428,7 @@ impl From for CandidateEntry { } } -impl From for crate::approval_db::v2::CandidateEntry { +impl From for crate::approval_db::v3::CandidateEntry { fn from(entry: CandidateEntry) -> Self { Self { candidate: entry.candidate, @@ -614,8 +654,8 @@ impl BlockEntry { } } -impl From for BlockEntry { - fn from(entry: crate::approval_db::v2::BlockEntry) -> Self { +impl From for BlockEntry { + fn from(entry: crate::approval_db::v3::BlockEntry) -> Self { BlockEntry { block_hash: entry.block_hash, parent_hash: entry.parent_hash, @@ -654,7 +694,25 @@ impl From for BlockEntry { } } -impl From for crate::approval_db::v2::BlockEntry { +impl From for BlockEntry { + fn from(entry: crate::approval_db::v2::BlockEntry) -> Self { + BlockEntry { + block_hash: entry.block_hash, + parent_hash: entry.parent_hash, + block_number: entry.block_number, + session: entry.session, + slot: entry.slot, + relay_vrf_story: RelayVRFStory(entry.relay_vrf_story), + candidates: entry.candidates, + approved_bitfield: entry.approved_bitfield, + children: entry.children, + distributed_assignments: entry.distributed_assignments, + candidates_pending_signature: Default::default(), + } + } +} + +impl From for crate::approval_db::v3::BlockEntry { fn from(entry: BlockEntry) -> Self { Self { block_hash: entry.block_hash, @@ -676,8 +734,8 @@ impl From for crate::approval_db::v2::BlockEntry { } } -impl From for CandidateSigningContext { - fn from(signing_context: crate::approval_db::v2::CandidateSigningContext) -> Self { +impl From for CandidateSigningContext { + fn from(signing_context: crate::approval_db::v3::CandidateSigningContext) -> Self { Self { candidate_hash: signing_context.candidate_hash, sign_no_later_than_tick: signing_context.sign_no_later_than_tick.into(), @@ -685,7 +743,7 @@ impl From for CandidateSigningC } } -impl From for crate::approval_db::v2::CandidateSigningContext { +impl From for crate::approval_db::v3::CandidateSigningContext { fn from(signing_context: CandidateSigningContext) -> Self { Self { candidate_hash: signing_context.candidate_hash, diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 04aef90aae97..4b78e990e660 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -56,7 +56,7 @@ use std::{ }; use super::{ - approval_db::v2::StoredBlockRange, + approval_db::common::StoredBlockRange, backend::BackendWriteOp, import::tests::{ garbage_vrf_signature, AllowedSlots, BabeEpoch, BabeEpochConfiguration, @@ -116,7 +116,7 @@ fn make_sync_oracle(val: bool) -> (Box, TestSyncOracleHan #[cfg(test)] pub mod test_constants { - use crate::approval_db::v2::Config as DatabaseConfig; + use crate::approval_db::common::Config as DatabaseConfig; const DATA_COL: u32 = 0; pub(crate) const NUM_COLUMNS: u32 = 1; diff --git a/polkadot/node/service/src/parachains_db/upgrade.rs b/polkadot/node/service/src/parachains_db/upgrade.rs index 1d76c79d3e32..d22eebb5c8d4 100644 --- a/polkadot/node/service/src/parachains_db/upgrade.rs +++ b/polkadot/node/service/src/parachains_db/upgrade.rs @@ -20,10 +20,16 @@ use std::{ fs, io, path::{Path, PathBuf}, str::FromStr, + sync::Arc, }; -use polkadot_node_core_approval_voting::approval_db::v2::{ - migration_helpers::v1_to_v2, Config as ApprovalDbConfig, +use polkadot_node_core_approval_voting::approval_db::{ + common::{Config as ApprovalDbConfig, Result as ApprovalDbResult}, + v2::migration_helpers::v1_to_latest, + v3::migration_helpers::v2_to_latest, +}; +use polkadot_node_subsystem_util::database::{ + kvdb_impl::DbAdapter as RocksDbAdapter, paritydb_impl::DbAdapter as ParityDbAdapter, Database, }; type Version = u32; @@ -32,7 +38,9 @@ const VERSION_FILE_NAME: &'static str = "parachain_db_version"; /// Current db version. /// Version 4 changes approval db format for `OurAssignment`. -pub(crate) const CURRENT_VERSION: Version = 4; +/// Version 5 changes approval db format to hold some additional +/// information about delayed approvals. +pub(crate) const CURRENT_VERSION: Version = 5; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -101,7 +109,8 @@ pub(crate) fn try_upgrade_db_to_next_version( // 2 -> 3 migration Some(2) => migrate_from_version_2_to_3(db_path, db_kind)?, // 3 -> 4 migration - Some(3) => migrate_from_version_3_to_4(db_path, db_kind)?, + Some(3) => migrate_from_version_3_or_4_to_5(db_path, db_kind, v1_to_latest)?, + Some(4) => migrate_from_version_3_or_4_to_5(db_path, db_kind, v2_to_latest)?, // Already at current version, do nothing. Some(CURRENT_VERSION) => CURRENT_VERSION, // This is an arbitrary future version, we don't handle it. @@ -174,14 +183,19 @@ fn migrate_from_version_1_to_2(path: &Path, db_kind: DatabaseKind) -> Result Result { +fn migrate_from_version_3_or_4_to_5( + path: &Path, + db_kind: DatabaseKind, + migration_function: F, +) -> Result +where + F: Fn(Arc, ApprovalDbConfig) -> ApprovalDbResult<()>, +{ gum::info!(target: LOG_TARGET, "Migrating parachains db from version 3 to version 4 ..."); - use polkadot_node_subsystem_util::database::{ - kvdb_impl::DbAdapter as RocksDbAdapter, paritydb_impl::DbAdapter as ParityDbAdapter, - }; - use std::sync::Arc; let approval_db_config = ApprovalDbConfig { col_approval_data: super::REAL_COLUMNS.col_approval_data }; @@ -194,7 +208,8 @@ fn migrate_from_version_3_to_4(path: &Path, db_kind: DatabaseKind) -> Result { let db_path = path @@ -207,7 +222,8 @@ fn migrate_from_version_3_to_4(path: &Path, db_kind: DatabaseKind) -> Result Date: Tue, 31 Oct 2023 11:02:07 +0200 Subject: [PATCH 54/89] Modify the way we are doing the sampling See https://github.com/paritytech/polkadot-sdk/pull/1178#discussion_r1375786733 for more details Signed-off-by: Alexandru Gheorghe --- Cargo.lock | 2 + polkadot/node/core/approval-voting/Cargo.toml | 2 + .../node/core/approval-voting/src/criteria.rs | 63 +++++++++++++------ 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a096faf19a3b..43f72366593f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11940,6 +11940,8 @@ dependencies = [ "polkadot-overseer", "polkadot-primitives", "polkadot-primitives-test-helpers", + "rand 0.8.5", + "rand_chacha 0.3.1", "rand_core 0.5.1", "sc-keystore", "schnellru", diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index b89d7d2cedc0..f7ea1d210777 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -32,6 +32,8 @@ sp-consensus-slots = { path = "../../../../substrate/primitives/consensus/slots" sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto", default-features = false, features = ["full_crypto"] } sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } rand_core = "0.5.1" +rand_chacha = { version = "0.3.1" } +rand = "0.8.5" [dev-dependencies] async-trait = "0.1.57" diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 1f751e2bf140..0b69afc53280 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -16,6 +16,7 @@ //! Assignment criteria VRF generation and checking. +use itertools::Itertools; use parity_scale_codec::{Decode, Encode}; use polkadot_node_primitives::approval::{ self as approval_types, @@ -26,14 +27,18 @@ use polkadot_primitives::{ AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, ValidatorIndex, }; +use rand::Rng; +use rand_chacha::{rand_core::SeedableRng, ChaCha20Rng}; use sc_keystore::LocalKeystore; use sp_application_crypto::ByteArray; use merlin::Transcript; use schnorrkel::vrf::VRFInOut; -use itertools::Itertools; -use std::collections::{hash_map::Entry, HashMap}; +use std::{ + cmp::min, + collections::{hash_map::Entry, HashMap, HashSet}, +}; use super::LOG_TARGET; @@ -125,17 +130,11 @@ fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript /// A hard upper bound on num_cores * target_checkers / num_validators const MAX_MODULO_SAMPLES: usize = 40; -use std::convert::AsMut; +/// The maximum number of samples we take to try to obtain an +/// a distinct core_index. +const MAX_SAMPLING_ITERATIONS: usize = 20; -fn clone_into_array(slice: &[T]) -> A -where - A: Default + AsMut<[T]>, - T: Clone, -{ - let mut a = A::default(); - >::as_mut(&mut a).clone_from_slice(slice); - a -} +use std::convert::AsMut; struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); @@ -170,14 +169,38 @@ fn relay_vrf_modulo_cores( ); } - vrf_in_out - .make_bytes::(approval_types::v2::CORE_RANDOMNESS_CONTEXT) - .0 - .chunks_exact(4) - .take(num_samples as usize) - .map(move |sample| CoreIndex(u32::from_le_bytes(clone_into_array(&sample)) % max_cores)) - .unique() - .collect::>() + if 2 * num_samples > max_cores { + gum::error!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "Suboptimal configuration `num_samples` should be less than `n_cores` / 2", + ); + } + + let mut rand_chacha = + ChaCha20Rng::from_seed(vrf_in_out.make_bytes::<::Seed>( + approval_types::v2::CORE_RANDOMNESS_CONTEXT, + )); + + let mut res = HashSet::with_capacity(num_samples as usize); + while res.len() < min(num_samples, max_cores) as usize { + // Production networks should run with num_samples no larger than `n_cores/2`, so on worst + // case on each iteration this loop has >1/2 chance of finishing. + // With our current production parameters the chances for each iteration not hitting + // a duplicate are around 90%. + // See: https://github.com/paritytech/polkadot-sdk/pull/1178#discussion_r1376501206 + // for more details. + for _ in 0..MAX_SAMPLING_ITERATIONS { + let random_core = rand_chacha.gen_range(0..max_cores).into(); + if !res.contains(&random_core) { + res.insert(random_core); + break + } + } + } + res.into_iter().collect_vec() } fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { From 64f2195f45736c3718d824a4068e4894513dbdc6 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 31 Oct 2023 12:51:09 +0200 Subject: [PATCH 55/89] Fixup check_rejects_delay_bad_vrf Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 0b69afc53280..01fff31a4be0 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -1135,7 +1135,7 @@ mod tests { #[test] fn check_rejects_delay_bad_vrf() { - check_mutated_assignments(40, 10, 8, |m| { + check_mutated_assignments(40, 100, 8, |m| { let vrf_signature = garbage_vrf_signature(); match m.cert.kind.clone() { AssignmentCertKindV2::RelayVRFDelay { .. } => { From 04798fa7b8edd76ae4431b468f8ab1dc6cf4e81e Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 31 Oct 2023 16:01:44 +0200 Subject: [PATCH 56/89] Address some trivial review feedback Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting/src/approval_db/v3/mod.rs | 2 +- .../node/core/approval-voting/src/persisted_entries.rs | 2 +- polkadot/node/network/approval-distribution/src/lib.rs | 2 +- polkadot/primitives/src/v6/mod.rs | 5 ++++- .../src/node/approval/approval-voting.md | 6 +++--- .../roadmap/implementers-guide/src/protocol-approval.md | 8 ++++---- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs index 4483a4740a1f..031a51edd64b 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs @@ -97,7 +97,7 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, - // A list of candidates that has been approved, but we didn't not sign and + // A list of candidates we have checked, but didn't not sign and // advertise the vote yet. pub candidates_pending_signature: BTreeMap, // Assignments we already distributed. A 1 bit means the candidate index for which diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 05da6f844063..ac3b704c31c2 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -461,7 +461,7 @@ pub struct BlockEntry { // block. The block can be considered approved if the bitfield has all bits set to `true`. pub approved_bitfield: Bitfield, pub children: Vec, - // A list of candidates that has been approved, but we didn't not sign and + // A list of candidates we have checked, but didn't not sign and // advertise the vote yet. candidates_pending_signature: BTreeMap, // A list of assignments for which we already distributed the assignment. diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index d5188ef52ec7..91e850ad5b31 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -2606,7 +2606,7 @@ pub(crate) async fn send_approvals_batched( .clone() .into_iter() .filter(|approval| approval.candidate_indices.count_ones() == 1) - .map(|val| val.try_into().expect("We checked conversion should succeed; qed")) + .filter_map(|val| val.try_into().ok()) .peekable(); while batches.peek().is_some() { diff --git a/polkadot/primitives/src/v6/mod.rs b/polkadot/primitives/src/v6/mod.rs index 0d9a85e1fd81..8f11092b8455 100644 --- a/polkadot/primitives/src/v6/mod.rs +++ b/polkadot/primitives/src/v6/mod.rs @@ -1070,7 +1070,7 @@ impl ApprovalVote { } } -/// A vote of approvalf for multiple candidates. +/// A vote of approval for multiple candidates. #[derive(Clone, RuntimeDebug)] pub struct ApprovalVoteMultipleCandidates<'a>(pub &'a Vec); @@ -1266,6 +1266,9 @@ pub enum DisputeStatement { impl DisputeStatement { /// Get the payload data for this type of dispute statement. + /// + /// Returns Error if the candidate_hash is not included in the list of signed + /// candidate from ApprovalCheckingMultipleCandidate. pub fn payload_data( &self, candidate_hash: CandidateHash, diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index 8d7daa173abc..345b3d2e6970 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -5,8 +5,8 @@ aims of this subsystem. Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to indicate intent to check a candidate. Upon successfully checking, they don't immediately send the vote instead -they queue the check for a short period of time `MAX_APPROVALS_COALESCE_TICKS` to give the opportunity of the -validator to vote for more than one candidate. Once MAX_APPROVALS_COALESCE_TICKS have passed or at least +they queue the check for a short period of time `MAX_APPROVAL_COALESCE_WAIT_TICKS` to give the opportunity of the +validator to vote for more than one candidate. Once MAX_APPROVAL_COALESCE_WAIT_TICKS have passed or at least `MAX_APPROVAL_COALESCE_COUNT` are ready they broadcast an approval vote for all candidates. If a validator doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being prevented from recovering or validating the block data and that more validators should self-select to @@ -123,7 +123,7 @@ struct BlockEntry { // this block. The block can be considered approved has all bits set to 1 approved_bitfield: Bitfield, children: Vec, - // A list of candidates that has been approved, but we didn't not sign and + // A list of candidates we have checked, but didn't not sign and // advertise the vote yet. candidates_pending_signature: BTreeMap, // Assignments we already distributed. A 1 bit means the candidate index for which diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index 1a217afb9b71..a3427831753d 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -300,11 +300,11 @@ TODO: When? Is this optimal for the network? etc. To reduce the necessary network bandwidth and cpu time when a validator has more than one candidate to approve we are doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we -delay the sending of it untill one of three things happen: +delay sending it until one of three things happen: - We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we are ready to vote for. -- `MAX_APPROVALS_COALESCE_TICKS` have passed since the we were ready to approve the candidate. -- We are already in the last third of the now-show period in order to avoid creating accidental no shows, which in - turn my trigger other assignments. +- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since the we were ready to approve the candidate. +- We are already in the last third of the no-show period in order to avoid creating accidental no-shows, which in + turn might trigger other assignments. ## On-chain verification From 440c38be89816566fa45ba03abcf60d23738fe6c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 31 Oct 2023 17:10:27 +0200 Subject: [PATCH 57/89] Use VersionedMigration Signed-off-by: Alexandru Gheorghe --- .../src/configuration/migration/v10.rs | 45 ++++++++++--------- polkadot/runtime/rococo/src/lib.rs | 2 +- polkadot/runtime/westend/src/lib.rs | 2 +- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/polkadot/runtime/parachains/src/configuration/migration/v10.rs b/polkadot/runtime/parachains/src/configuration/migration/v10.rs index 53c5f4d2c52c..ea6624e99743 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v10.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v10.rs @@ -18,13 +18,10 @@ use crate::configuration::{self, Config, Pallet}; use frame_support::{ - pallet_prelude::*, - traits::{Defensive, StorageVersion}, - weights::Weight, + migrations::VersionedMigration, pallet_prelude::*, traits::Defensive, weights::Weight, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::{vstaging::ApprovalVotingParams, SessionIndex, LEGACY_MIN_BACKING_VOTES}; -use sp_runtime::Perbill; +use primitives::{vstaging::ApprovalVotingParams, SessionIndex}; use sp_std::vec::Vec; use frame_support::traits::OnRuntimeUpgrade; @@ -62,8 +59,16 @@ mod v10 { >; } -pub struct MigrateToV10(sp_std::marker::PhantomData); -impl OnRuntimeUpgrade for MigrateToV10 { +pub type MigrateV9ToV10 = VersionedMigration< + 9, + 10, + UncheckedMigrateToV10, + Pallet, + ::DbWeight, +>; + +pub struct UncheckedMigrateToV10(sp_std::marker::PhantomData); +impl OnRuntimeUpgrade for UncheckedMigrateToV10 { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV10"); @@ -72,17 +77,11 @@ impl OnRuntimeUpgrade for MigrateToV10 { fn on_runtime_upgrade() -> Weight { log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 started"); - if StorageVersion::get::>() == 9 { - let weight_consumed = migrate_to_v10::(); + let weight_consumed = migrate_to_v10::(); - log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 executed successfully"); - StorageVersion::new(10).put::>(); + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 executed successfully"); - weight_consumed - } else { - log::warn!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 should be removed."); - T::DbWeight::get().reads(1) - } + weight_consumed } #[cfg(feature = "try-runtime")] @@ -145,12 +144,12 @@ pvf_voting_ttl : pre.pvf_voting_ttl, minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, async_backing_params : pre.async_backing_params, executor_params : pre.executor_params, -on_demand_queue_max_size : 10_000u32, -on_demand_base_fee : 10_000_000u128, -on_demand_fee_variability : Perbill::from_percent(3), -on_demand_target_queue_utilization : Perbill::from_percent(25), -on_demand_ttl : 5u32.into(), -minimum_backing_votes : LEGACY_MIN_BACKING_VOTES, +on_demand_queue_max_size : pre.on_demand_queue_max_size, +on_demand_base_fee : pre.on_demand_base_fee, +on_demand_fee_variability : pre.on_demand_fee_variability, +on_demand_target_queue_utilization : pre.on_demand_target_queue_utilization, +on_demand_ttl : pre.on_demand_ttl, +minimum_backing_votes : pre.minimum_backing_votes, approval_voting_params : ApprovalVotingParams { max_approval_coalesce_count: 1, } @@ -179,6 +178,8 @@ approval_voting_params : ApprovalVotingParams { #[cfg(test)] mod tests { + use primitives::LEGACY_MIN_BACKING_VOTES; + use super::*; use crate::mock::{new_test_ext, Test}; diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 0a7322e0c058..de66fa0098c1 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1484,7 +1484,7 @@ pub mod migrations { frame_support::migrations::RemovePallet::DbWeight>, frame_support::migrations::RemovePallet::DbWeight>, frame_support::migrations::RemovePallet::DbWeight>, - parachains_configuration::migration::v10::MigrateToV10, + parachains_configuration::migration::v10::MigrateV9ToV10, ); } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index db3cf2b14e00..30fbda5d9c4f 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1549,7 +1549,7 @@ pub mod migrations { pallet_nomination_pools::migration::versioned_migrations::V5toV6, pallet_referenda::migration::v1::MigrateV0ToV1, pallet_nomination_pools::migration::versioned_migrations::V6ToV7, - parachains_configuration::migration::v10::MigrateToV10, + parachains_configuration::migration::v10::MigrateV9ToV10, ); } From 569ebb7ee5a9110d347fb47ca2336468d47de3b0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 31 Oct 2023 18:38:13 +0200 Subject: [PATCH 58/89] Do not accept ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates until coalescing is not enabled Signed-off-by: Alexandru Gheorghe --- polkadot/runtime/parachains/src/disputes.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/disputes.rs b/polkadot/runtime/parachains/src/disputes.rs index e0ff68f79063..fce5dc24d28d 100644 --- a/polkadot/runtime/parachains/src/disputes.rs +++ b/polkadot/runtime/parachains/src/disputes.rs @@ -952,6 +952,8 @@ impl Pallet { None => return StatementSetFilter::RemoveAll, }; + let config = >::config(); + let n_validators = session_info.validators.len(); // Check for ancient. @@ -1015,6 +1017,7 @@ impl Pallet { set.session, statement, signature, + config.approval_voting_params.max_approval_coalesce_count > 1, ) { log::warn!("Failed to check dispute signature"); @@ -1262,6 +1265,7 @@ fn check_signature( session: SessionIndex, statement: &DisputeStatement, validator_signature: &ValidatorSignature, + approval_multiple_candidates_enabled: bool, ) -> Result<(), ()> { let payload = match statement { DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => @@ -1281,7 +1285,7 @@ fn check_signature( DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates( candidates, )) => - if candidates.contains(&candidate_hash) { + if approval_multiple_candidates_enabled && candidates.contains(&candidate_hash) { ApprovalVoteMultipleCandidates(candidates).signing_payload(session) } else { return Err(()) From ac2271b43ea2638c69468be3884ead95a7852410 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 1 Nov 2023 10:02:28 +0200 Subject: [PATCH 59/89] Make approval_voting_params session buferred Signed-off-by: Alexandru Gheorghe --- .../src/blockchain_rpc_client.rs | 11 +- .../src/rpc_client.rs | 1 + .../node/core/approval-voting/src/import.rs | 1 - polkadot/node/core/approval-voting/src/lib.rs | 116 ++++++------------ .../node/core/approval-voting/src/tests.rs | 17 ++- polkadot/node/core/runtime-api/src/cache.rs | 34 ++--- polkadot/node/core/runtime-api/src/lib.rs | 13 +- polkadot/node/subsystem-types/src/messages.rs | 5 +- .../subsystem-types/src/runtime_client.rs | 20 ++- 9 files changed, 99 insertions(+), 119 deletions(-) diff --git a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs index 43d2adf8171a..cabf9ef73c28 100644 --- a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs +++ b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs @@ -367,8 +367,15 @@ impl RuntimeApiSubsystemClient for BlockChainRpcClient { } /// Approval voting configuration parameters - async fn approval_voting_params(&self, at: Hash) -> Result { - Ok(self.rpc_client.parachain_host_staging_approval_voting_params(at).await?) + async fn approval_voting_params( + &self, + at: Hash, + session_index: polkadot_primitives::SessionIndex, + ) -> Result { + Ok(self + .rpc_client + .parachain_host_staging_approval_voting_params(at, session_index) + .await?) } } diff --git a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs index 4418f4b53002..e927c37e559a 100644 --- a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs +++ b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs @@ -620,6 +620,7 @@ impl RelayChainRpcClient { pub async fn parachain_host_staging_approval_voting_params( &self, at: RelayHash, + _session_index: SessionIndex, ) -> Result { self.call_remote_runtime_function( "ParachainHost_staging_approval_voting_params", diff --git a/polkadot/node/core/approval-voting/src/import.rs b/polkadot/node/core/approval-voting/src/import.rs index ba894c2a30ff..0379b5ad8c15 100644 --- a/polkadot/node/core/approval-voting/src/import.rs +++ b/polkadot/node/core/approval-voting/src/import.rs @@ -645,7 +645,6 @@ pub(crate) mod tests { clock: Box::new(MockClock::default()), assignment_criteria: Box::new(MockAssignmentCriteria), spans: HashMap::new(), - approval_voting_params_cache: None, } } diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index c12ba99fc319..ac32aa957068 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -124,10 +124,6 @@ pub(crate) const LOG_TARGET: &str = "parachain::approval-voting"; // The max number of ticks we delay sending the approval after we are ready to issue the approval const MAX_APPROVAL_COALESCE_WAIT_TICKS: Tick = 12; -// The maximum approval params we cache locally in the subsytem, so that we don't have -// to do the back and forth to the runtime subsystem api. -const APPROVAL_PARAMS_CACHE_SIZE: u32 = 128; - /// Configuration for the approval voting subsystem #[derive(Debug, Clone)] pub struct Config { @@ -163,9 +159,6 @@ pub struct ApprovalVotingSubsystem { db: Arc, mode: Mode, metrics: Metrics, - // Store approval-voting params, so that we don't to always go to RuntimeAPI - // to ask this information which requires a task context switch. - approval_voting_params_cache: Option>, } #[derive(Clone)] @@ -451,24 +444,6 @@ impl ApprovalVotingSubsystem { keystore: Arc, sync_oracle: Box, metrics: Metrics, - ) -> Self { - Self::with_config_and_cache( - config, - db, - keystore, - sync_oracle, - metrics, - Some(LruMap::new(ByLength::new(APPROVAL_PARAMS_CACHE_SIZE))), - ) - } - - pub fn with_config_and_cache( - config: Config, - db: Arc, - keystore: Arc, - sync_oracle: Box, - metrics: Metrics, - approval_voting_params_cache: Option>, ) -> Self { ApprovalVotingSubsystem { keystore, @@ -477,7 +452,6 @@ impl ApprovalVotingSubsystem { db_config: DatabaseConfig { col_approval_data: config.col_approval_data }, mode: Mode::Syncing(sync_oracle), metrics, - approval_voting_params_cache, } } @@ -798,9 +772,6 @@ struct State { clock: Box, assignment_criteria: Box, spans: HashMap, - // Store approval-voting params, so that we don't to always go to RuntimeAPI - // to ask this information which requires a task context switch. - approval_voting_params_cache: Option>, } #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] @@ -866,49 +837,45 @@ impl State { // To avoid crossing the subsystem boundary every-time we are caching locally the values. #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn get_approval_voting_params_or_default( - &mut self, + &self, ctx: &mut Context, + session_index: SessionIndex, block_hash: Hash, ) -> ApprovalVotingParams { - if let Some(params) = self - .approval_voting_params_cache - .as_mut() - .and_then(|cache| cache.get(&block_hash)) - { - *params - } else { - let (s_tx, s_rx) = oneshot::channel(); + let (s_tx, s_rx) = oneshot::channel(); - ctx.send_message(RuntimeApiMessage::Request( - block_hash, - RuntimeApiRequest::ApprovalVotingParams(s_tx), - )) - .await; + ctx.send_message(RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::ApprovalVotingParams(session_index, s_tx), + )) + .await; - match s_rx.await { - Ok(Ok(params)) => { - self.approval_voting_params_cache - .as_mut() - .map(|cache| cache.insert(block_hash, params)); - params - }, - Ok(Err(err)) => { - gum::error!( - target: LOG_TARGET, - ?err, - "Could not request approval voting params from runtime using defaults" - ); - ApprovalVotingParams { max_approval_coalesce_count: 1 } - }, - Err(err) => { - gum::error!( - target: LOG_TARGET, - ?err, - "Could not request approval voting params from runtime using defaults" - ); - ApprovalVotingParams { max_approval_coalesce_count: 1 } - }, - } + match s_rx.await { + Ok(Ok(params)) => { + gum::trace!( + target: LOG_TARGET, + approval_voting_params = ?params, + session = ?session_index, + "Using the following subsystem params" + ); + params + }, + Ok(Err(err)) => { + gum::error!( + target: LOG_TARGET, + ?err, + "Could not request approval voting params from runtime using defaults" + ); + ApprovalVotingParams { max_approval_coalesce_count: 1 } + }, + Err(err) => { + gum::error!( + target: LOG_TARGET, + ?err, + "Could not request approval voting params from runtime using defaults" + ); + ApprovalVotingParams { max_approval_coalesce_count: 1 } + }, } } } @@ -960,7 +927,6 @@ where clock, assignment_criteria, spans: HashMap::new(), - approval_voting_params_cache: subsystem.approval_voting_params_cache.take(), }; // `None` on start-up. Gets initialized/updated on leaf update @@ -1056,13 +1022,11 @@ where "Sign approval for multiple candidates", ); - let approval_params = state.get_approval_voting_params_or_default(&mut ctx, block_hash).await; - match maybe_create_signature( &mut overlayed_db, &mut session_info_provider, - approval_params, - &state, &mut ctx, + &state, + &mut ctx, block_hash, validator_index, &subsystem.metrics, @@ -3376,8 +3340,6 @@ async fn issue_approval( "Ready to issue approval vote", ); - let approval_params = state.get_approval_voting_params_or_default(ctx, block_hash).await; - let actions = advance_approval_state( ctx.sender(), state, @@ -3394,7 +3356,6 @@ async fn issue_approval( if let Some(next_wakeup) = maybe_create_signature( db, session_info_provider, - approval_params, state, ctx, block_hash, @@ -3418,7 +3379,6 @@ async fn issue_approval( async fn maybe_create_signature( db: &mut OverlayedBackend<'_, impl Backend>, session_info_provider: &mut RuntimeInfo, - approval_params: ApprovalVotingParams, state: &State, ctx: &mut Context, block_hash: Hash, @@ -3438,6 +3398,10 @@ async fn maybe_create_signature( }, }; + let approval_params = state + .get_approval_voting_params_or_default(ctx, block_entry.session(), block_hash) + .await; + gum::trace!( target: LOG_TARGET, "Candidates pending signatures {:}", block_entry.num_candidates_pending_signature() diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 4b78e990e660..a27315d416e3 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -542,7 +542,7 @@ fn test_harness>( let subsystem = run( context, - ApprovalVotingSubsystem::with_config_and_cache( + ApprovalVotingSubsystem::with_config( Config { col_approval_data: test_constants::TEST_CONFIG.col_approval_data, slot_duration_millis: SLOT_DURATION_MILLIS, @@ -551,7 +551,6 @@ fn test_harness>( Arc::new(keystore), sync_oracle, Metrics::default(), - None, ), clock.clone(), assignment_criteria, @@ -2814,7 +2813,7 @@ async fn handle_double_assignment_import( assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 1, })); @@ -2828,7 +2827,7 @@ async fn handle_double_assignment_import( assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 1, })); @@ -3751,7 +3750,7 @@ async fn handle_approval_on_max_coalesce_count( assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 2, })); @@ -3760,7 +3759,7 @@ async fn handle_approval_on_max_coalesce_count( assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 2, })); @@ -3816,7 +3815,7 @@ async fn handle_approval_on_max_wait_time( // First time we fetch the configuration when we are ready to approve the first candidate assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: MAX_COALESCE_COUNT, })); @@ -3826,7 +3825,7 @@ async fn handle_approval_on_max_wait_time( // Second time we fetch the configuration when we are ready to approve the second candidate assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: MAX_COALESCE_COUNT, })); @@ -3853,7 +3852,7 @@ async fn handle_approval_on_max_wait_time( // approval assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(sender))) => { + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { let _ = sender.send(Ok(ApprovalVotingParams { max_approval_coalesce_count: 3, })); diff --git a/polkadot/node/core/runtime-api/src/cache.rs b/polkadot/node/core/runtime-api/src/cache.rs index 83ebf6884890..0b21da919bf5 100644 --- a/polkadot/node/core/runtime-api/src/cache.rs +++ b/polkadot/node/core/runtime-api/src/cache.rs @@ -67,7 +67,7 @@ pub(crate) struct RequestResultCache { disabled_validators: LruMap>, para_backing_state: LruMap<(Hash, ParaId), Option>, async_backing_params: LruMap, - approval_voting_params: LruMap, + approval_voting_params: LruMap, } impl Default for RequestResultCache { @@ -398,21 +398,6 @@ impl RequestResultCache { self.disputes.insert(relay_parent, value); } - pub(crate) fn approval_voting_params( - &mut self, - relay_parent: &Hash, - ) -> Option<&ApprovalVotingParams> { - self.approval_voting_params.get(relay_parent).map(|v| &*v) - } - - pub(crate) fn cache_approval_voting_params( - &mut self, - relay_parent: Hash, - value: ApprovalVotingParams, - ) { - self.approval_voting_params.insert(relay_parent, value); - } - pub(crate) fn unapplied_slashes( &mut self, relay_parent: &Hash, @@ -507,6 +492,21 @@ impl RequestResultCache { ) { self.async_backing_params.insert(key, value); } + + pub(crate) fn approval_voting_params( + &mut self, + key: (Hash, SessionIndex), + ) -> Option<&ApprovalVotingParams> { + self.approval_voting_params.get(&key.1).map(|v| &*v) + } + + pub(crate) fn cache_approval_voting_params( + &mut self, + session_index: SessionIndex, + value: ApprovalVotingParams, + ) { + self.approval_voting_params.insert(session_index, value); + } } pub(crate) enum RequestResult { @@ -554,7 +554,7 @@ pub(crate) enum RequestResult { slashing::OpaqueKeyOwnershipProof, Option<()>, ), - ApprovalVotingParams(Hash, ApprovalVotingParams), + ApprovalVotingParams(Hash, SessionIndex, ApprovalVotingParams), DisabledValidators(Hash, Vec), ParaBackingState(Hash, ParaId, Option), AsyncBackingParams(Hash, async_backing::AsyncBackingParams), diff --git a/polkadot/node/core/runtime-api/src/lib.rs b/polkadot/node/core/runtime-api/src/lib.rs index bc6dc8722876..36ec8325f8aa 100644 --- a/polkadot/node/core/runtime-api/src/lib.rs +++ b/polkadot/node/core/runtime-api/src/lib.rs @@ -165,8 +165,8 @@ where KeyOwnershipProof(relay_parent, validator_id, key_ownership_proof) => self .requests_cache .cache_key_ownership_proof((relay_parent, validator_id), key_ownership_proof), - RequestResult::ApprovalVotingParams(relay_parent, params) => - self.requests_cache.cache_approval_voting_params(relay_parent, params), + RequestResult::ApprovalVotingParams(_relay_parent, session_index, params) => + self.requests_cache.cache_approval_voting_params(session_index, params), SubmitReportDisputeLost(_, _, _, _) => {}, DisabledValidators(relay_parent, disabled_validators) => self.requests_cache.cache_disabled_validators(relay_parent, disabled_validators), @@ -300,8 +300,9 @@ where Request::SubmitReportDisputeLost(dispute_proof, key_ownership_proof, sender) }, ), - Request::ApprovalVotingParams(sender) => query!(approval_voting_params(), sender) - .map(|sender| Request::ApprovalVotingParams(sender)), + Request::ApprovalVotingParams(session_index, sender) => + query!(approval_voting_params(session_index), sender) + .map(|sender| Request::ApprovalVotingParams(session_index, sender)), Request::DisabledValidators(sender) => query!(disabled_validators(), sender) .map(|sender| Request::DisabledValidators(sender)), Request::ParaBackingState(para, sender) => query!(para_backing_state(para), sender) @@ -561,10 +562,10 @@ where ver = Request::KEY_OWNERSHIP_PROOF_RUNTIME_REQUIREMENT, sender ), - Request::ApprovalVotingParams(sender) => { + Request::ApprovalVotingParams(session_index, sender) => { query!( ApprovalVotingParams, - approval_voting_params(), + approval_voting_params(session_index), ver = Request::APPROVAL_VOTING_PARAMS_REQUIREMENT, sender ) diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index c5be711ea301..6b18e3e48209 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -696,8 +696,6 @@ pub enum RuntimeApiRequest { slashing::OpaqueKeyOwnershipProof, RuntimeApiSender>, ), - /// Approval voting params - ApprovalVotingParams(RuntimeApiSender), /// Get the minimum required backing votes. MinimumBackingVotes(SessionIndex, RuntimeApiSender), /// Returns all disabled validators at a given block height. @@ -708,6 +706,9 @@ pub enum RuntimeApiRequest { /// /// If it's not supported by the Runtime, the async backing is said to be disabled. AsyncBackingParams(RuntimeApiSender), + /// Approval voting params + /// `V9` + ApprovalVotingParams(SessionIndex, RuntimeApiSender), } impl RuntimeApiRequest { diff --git a/polkadot/node/subsystem-types/src/runtime_client.rs b/polkadot/node/subsystem-types/src/runtime_client.rs index e48dc046896d..4bb383f4107d 100644 --- a/polkadot/node/subsystem-types/src/runtime_client.rs +++ b/polkadot/node/subsystem-types/src/runtime_client.rs @@ -262,7 +262,11 @@ pub trait RuntimeApiSubsystemClient { // == v9: Approval voting params == /// Approval voting configuration parameters - async fn approval_voting_params(&self, at: Hash) -> Result; + async fn approval_voting_params( + &self, + at: Hash, + session_index: SessionIndex, + ) -> Result; } /// Default implementation of [`RuntimeApiSubsystemClient`] using the client. @@ -489,11 +493,6 @@ where runtime_api.submit_report_dispute_lost(at, dispute_proof, key_ownership_proof) } - /// Approval voting configuration parameters - async fn approval_voting_params(&self, at: Hash) -> Result { - self.client.runtime_api().approval_voting_params(at) - } - async fn minimum_backing_votes( &self, at: Hash, @@ -520,4 +519,13 @@ where async fn disabled_validators(&self, at: Hash) -> Result, ApiError> { self.client.runtime_api().disabled_validators(at) } + + /// Approval voting configuration parameters + async fn approval_voting_params( + &self, + at: Hash, + _session_index: SessionIndex, + ) -> Result { + self.client.runtime_api().approval_voting_params(at) + } } From 35cbc0ec465d314439bfbc21ab4431960bfad062 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 1 Nov 2023 12:11:31 +0200 Subject: [PATCH 60/89] Make clippy happy Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/runtime-api/src/tests.rs | 6 +- .../runtime/parachains/src/disputes/tests.rs | 126 ++++++++++++------ 2 files changed, 89 insertions(+), 43 deletions(-) diff --git a/polkadot/node/core/runtime-api/src/tests.rs b/polkadot/node/core/runtime-api/src/tests.rs index 5207d03a412d..79bdbf4f0080 100644 --- a/polkadot/node/core/runtime-api/src/tests.rs +++ b/polkadot/node/core/runtime-api/src/tests.rs @@ -243,7 +243,11 @@ impl RuntimeApiSubsystemClient for MockSubsystemClient { } /// Approval voting configuration parameters - async fn approval_voting_params(&self, _: Hash) -> Result { + async fn approval_voting_params( + &self, + _: Hash, + _: SessionIndex, + ) -> Result { todo!("Not required for tests") } diff --git a/polkadot/runtime/parachains/src/disputes/tests.rs b/polkadot/runtime/parachains/src/disputes/tests.rs index 0757084084f6..1f3f00132d68 100644 --- a/polkadot/runtime/parachains/src/disputes/tests.rs +++ b/polkadot/runtime/parachains/src/disputes/tests.rs @@ -1500,7 +1500,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_1 + &signed_1, + true, ) .is_ok()); assert!(check_signature( @@ -1508,7 +1509,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_1 + &signed_1, + true ) .is_err()); assert!(check_signature( @@ -1516,7 +1518,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_1, - &signed_1 + &signed_1, + true, ) .is_err()); assert!(check_signature( @@ -1524,7 +1527,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_1, - &signed_1 + &signed_1, + true ) .is_err()); assert!(check_signature( @@ -1532,7 +1536,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_1 + &signed_1, + true, ) .is_err()); assert!(check_signature( @@ -1540,7 +1545,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_1 + &signed_1, + true ) .is_err()); assert!(check_signature( @@ -1548,7 +1554,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_1 + &signed_1, + true ) .is_err()); assert!(check_signature( @@ -1556,7 +1563,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_1 + &signed_1, + true, ) .is_err()); @@ -1565,7 +1573,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_2 + &signed_2, + true, ) .is_ok()); assert!(check_signature( @@ -1573,7 +1582,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1581,7 +1591,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_2, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1589,7 +1600,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_2, - &signed_2 + &signed_2, + true ) .is_err()); assert!(check_signature( @@ -1597,7 +1609,8 @@ fn test_check_signature() { candidate_hash, session, &wrong_statement_2, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1605,7 +1618,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1613,7 +1627,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1621,7 +1636,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_2 + &signed_2, + true, ) .is_err()); assert!(check_signature( @@ -1629,7 +1645,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_2 + &signed_2, + true, ) .is_err()); @@ -1638,7 +1655,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_3 + &signed_3, + true, ) .is_ok()); assert!(check_signature( @@ -1646,7 +1664,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1654,7 +1673,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_3, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1662,7 +1682,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_3, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1670,7 +1691,8 @@ fn test_check_signature() { candidate_hash, session, &wrong_statement_3, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1678,7 +1700,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1686,7 +1709,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_3 + &signed_3, + true ) .is_err()); assert!(check_signature( @@ -1694,7 +1718,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_3 + &signed_3, + true, ) .is_err()); assert!(check_signature( @@ -1702,7 +1727,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_3 + &signed_3, + true, ) .is_err()); @@ -1711,7 +1737,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_4 + &signed_4, + true, ) .is_ok()); assert!(check_signature( @@ -1719,7 +1746,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1727,7 +1755,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_4, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1735,7 +1764,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_4, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1743,7 +1773,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1751,7 +1782,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1759,7 +1791,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_4 + &signed_4, + true, ) .is_err()); assert!(check_signature( @@ -1767,7 +1800,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_4 + &signed_4, + true, ) .is_err()); @@ -1776,7 +1810,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_5 + &signed_5, + true, ) .is_ok()); assert!(check_signature( @@ -1784,7 +1819,8 @@ fn test_check_signature() { candidate_hash, session, &statement_5, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1792,7 +1828,8 @@ fn test_check_signature() { wrong_candidate_hash, session, &statement_5, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1800,7 +1837,8 @@ fn test_check_signature() { candidate_hash, wrong_session, &statement_5, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1808,7 +1846,8 @@ fn test_check_signature() { candidate_hash, session, &statement_1, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1816,7 +1855,8 @@ fn test_check_signature() { candidate_hash, session, &statement_2, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1824,7 +1864,8 @@ fn test_check_signature() { candidate_hash, session, &statement_3, - &signed_5 + &signed_5, + true, ) .is_err()); assert!(check_signature( @@ -1832,7 +1873,8 @@ fn test_check_signature() { candidate_hash, session, &statement_4, - &signed_5 + &signed_5, + true, ) .is_err()); } From 3af951747d1a570ea9654fb513a5c7bf6e256a9a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 3 Nov 2023 10:47:58 +0200 Subject: [PATCH 61/89] Address review feedback in relay_vrf_modulo_cores Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting/src/criteria.rs | 81 ++++++++++++------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 01fff31a4be0..b5f4ad5c6194 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -27,8 +27,8 @@ use polkadot_primitives::{ AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, IndexedVec, SessionInfo, ValidatorIndex, }; -use rand::Rng; -use rand_chacha::{rand_core::SeedableRng, ChaCha20Rng}; +use rand::{seq::SliceRandom, SeedableRng}; +use rand_chacha::ChaCha20Rng; use sc_keystore::LocalKeystore; use sp_application_crypto::ByteArray; @@ -36,8 +36,8 @@ use merlin::Transcript; use schnorrkel::vrf::VRFInOut; use std::{ - cmp::min, - collections::{hash_map::Entry, HashMap, HashSet}, + cmp::{max, min}, + collections::{hash_map::Entry, HashMap}, }; use super::LOG_TARGET; @@ -130,10 +130,6 @@ fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript /// A hard upper bound on num_cores * target_checkers / num_validators const MAX_MODULO_SAMPLES: usize = 40; -/// The maximum number of samples we take to try to obtain an -/// a distinct core_index. -const MAX_SAMPLING_ITERATIONS: usize = 20; - use std::convert::AsMut; struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); @@ -165,7 +161,7 @@ fn relay_vrf_modulo_cores( n_cores = max_cores, num_samples, max_modulo_samples = MAX_MODULO_SAMPLES, - "`num_samples` is greater than `MAX_MODULO_SAMPLES`", + "`num_samples` is greater than `MAX_MODU`LO_SAMPLES`", ); } @@ -178,29 +174,30 @@ fn relay_vrf_modulo_cores( "Suboptimal configuration `num_samples` should be less than `n_cores` / 2", ); } - - let mut rand_chacha = + let rand_chacha = ChaCha20Rng::from_seed(vrf_in_out.make_bytes::<::Seed>( approval_types::v2::CORE_RANDOMNESS_CONTEXT, )); + generate_samples(rand_chacha, num_samples as usize, max_cores as usize) +} - let mut res = HashSet::with_capacity(num_samples as usize); - while res.len() < min(num_samples, max_cores) as usize { - // Production networks should run with num_samples no larger than `n_cores/2`, so on worst - // case on each iteration this loop has >1/2 chance of finishing. - // With our current production parameters the chances for each iteration not hitting - // a duplicate are around 90%. - // See: https://github.com/paritytech/polkadot-sdk/pull/1178#discussion_r1376501206 - // for more details. - for _ in 0..MAX_SAMPLING_ITERATIONS { - let random_core = rand_chacha.gen_range(0..max_cores).into(); - if !res.contains(&random_core) { - res.insert(random_core); - break - } - } - } - res.into_iter().collect_vec() +/// Generates `num_sumples` randomly from (0..max_cores) range +/// +/// Note! The algorithm can't change because validators on the other +/// side won't be able to check the assignments until they update. +/// This invariant is tested with `generate_samples_invariant`, so the +/// tests will catch any subtle changes in the implementation of this function +/// and its dependencies. +fn generate_samples( + mut rand_chacha: ChaCha20Rng, + num_samples: usize, + max_cores: usize, +) -> Vec { + let num_samples = max(1, min(num_samples, max_cores / 2)); + + let mut random_cores = (0..max_cores as u32).map(|val| val.into()).collect::>(); + let (samples, _) = random_cores.partial_shuffle(&mut rand_chacha, num_samples as usize); + samples.into_iter().map(|val| *val).collect_vec() } fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex { @@ -1209,4 +1206,32 @@ mod tests { } }); } + + #[test] + fn generate_samples_invariant() { + let seed = [ + 1, 0, 52, 0, 0, 0, 0, 0, 1, 0, 10, 0, 22, 32, 0, 0, 2, 0, 55, 49, 0, 11, 0, 0, 3, 0, 0, + 0, 0, 0, 2, 92, + ]; + let rand_chacha = ChaCha20Rng::from_seed(seed); + + let samples = generate_samples(rand_chacha.clone(), 6, 100); + let expected = vec![19, 79, 17, 75, 66, 30].into_iter().map(Into::into).collect_vec(); + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha.clone(), 6, 7); + let expected = vec![5, 4, 2].into_iter().map(Into::into).collect_vec(); + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha.clone(), 6, 12); + let expected = vec![2, 4, 7, 5, 11, 3].into_iter().map(Into::into).collect_vec(); + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha.clone(), 1, 100); + let expected = vec![30].into_iter().map(Into::into).collect_vec(); + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha, 0, 100); + assert_eq!(samples, expected); + } } From 1b84bb0d6e36aa453442156614186efc4e9191e0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 3 Nov 2023 15:31:31 +0200 Subject: [PATCH 62/89] Address more review feedback Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 11 +++++------ .../node/network/approval-distribution/src/lib.rs | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index ac32aa957068..f6d843a908e7 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -833,8 +833,7 @@ impl State { } } - // Returns the approval voting from the RuntimeApi. - // To avoid crossing the subsystem boundary every-time we are caching locally the values. + // Returns the approval voting params from the RuntimeApi. #[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] async fn get_approval_voting_params_or_default( &self, @@ -861,7 +860,7 @@ impl State { params }, Ok(Err(err)) => { - gum::error!( + gum::debug!( target: LOG_TARGET, ?err, "Could not request approval voting params from runtime using defaults" @@ -869,7 +868,7 @@ impl State { ApprovalVotingParams { max_approval_coalesce_count: 1 } }, Err(err) => { - gum::error!( + gum::debug!( target: LOG_TARGET, ?err, "Could not request approval voting params from runtime using defaults" @@ -3586,8 +3585,8 @@ fn compute_delayed_approval_sending_tick( let sign_no_later_than = min( tick_now + MAX_APPROVAL_COALESCE_WAIT_TICKS as Tick, - // We don't want to accidentally cause, no-shows so if we are past - // the seconnd half of the no show time, force the sending of the + // We don't want to accidentally cause no-shows, so if we are past + // the second half of the no show time, force the sending of the // approval immediately. assignment_triggered_tick + no_show_duration_ticks / 2, ); diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 91e850ad5b31..f30170b0f116 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -415,9 +415,8 @@ impl Knowledge { // Tells if all keys are contained by this peer_knowledge pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { - keys.iter().fold(true, |accumulator, assignment_key| { - accumulator && self.contains(&assignment_key.0, assignment_key.1) - }) + keys.iter() + .all(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) } } @@ -539,9 +538,9 @@ impl BlockEntry { // Tels if all candidate_indices are valid candidates pub fn contains_candidates(&self, candidate_indices: &CandidateBitfield) -> bool { - candidate_indices.iter_ones().fold(true, |accumulator, candidate_index| { - self.candidates.get(candidate_index as usize).is_some() && accumulator - }) + candidate_indices + .iter_ones() + .all(|candidate_index| self.candidates.get(candidate_index as usize).is_some()) } // Saves the given approval in all ApprovalEntries that contain an assignment for any of the // candidates in the approval. From 7a1c88c3ca3b29ed2c0cfcefc7689e5d48cc3a42 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 3 Nov 2023 16:40:58 +0200 Subject: [PATCH 63/89] Add few more tests Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting/src/criteria.rs | 56 +++++++++++-------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index b5f4ad5c6194..5e7718edb246 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -36,7 +36,7 @@ use merlin::Transcript; use schnorrkel::vrf::VRFInOut; use std::{ - cmp::{max, min}, + cmp::min, collections::{hash_map::Entry, HashMap}, }; @@ -155,25 +155,6 @@ fn relay_vrf_modulo_cores( // Configuration - `n_cores`. max_cores: u32, ) -> Vec { - if num_samples as usize > MAX_MODULO_SAMPLES { - gum::warn!( - target: LOG_TARGET, - n_cores = max_cores, - num_samples, - max_modulo_samples = MAX_MODULO_SAMPLES, - "`num_samples` is greater than `MAX_MODU`LO_SAMPLES`", - ); - } - - if 2 * num_samples > max_cores { - gum::error!( - target: LOG_TARGET, - n_cores = max_cores, - num_samples, - max_modulo_samples = MAX_MODULO_SAMPLES, - "Suboptimal configuration `num_samples` should be less than `n_cores` / 2", - ); - } let rand_chacha = ChaCha20Rng::from_seed(vrf_in_out.make_bytes::<::Seed>( approval_types::v2::CORE_RANDOMNESS_CONTEXT, @@ -193,7 +174,27 @@ fn generate_samples( num_samples: usize, max_cores: usize, ) -> Vec { - let num_samples = max(1, min(num_samples, max_cores / 2)); + if num_samples as usize > MAX_MODULO_SAMPLES { + gum::warn!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "`num_samples` is greater than `MAX_MODULO_SAMPLES`", + ); + } + + if 2 * num_samples > max_cores { + gum::error!( + target: LOG_TARGET, + n_cores = max_cores, + num_samples, + max_modulo_samples = MAX_MODULO_SAMPLES, + "Suboptimal configuration `num_samples` should be less than `n_cores` / 2", + ); + } + + let num_samples = min(MAX_MODULO_SAMPLES, min(num_samples, max_cores / 2)); let mut random_cores = (0..max_cores as u32).map(|val| val.into()).collect::>(); let (samples, _) = random_cores.partial_shuffle(&mut rand_chacha, num_samples as usize); @@ -1231,7 +1232,18 @@ mod tests { let expected = vec![30].into_iter().map(Into::into).collect_vec(); assert_eq!(samples, expected); - let samples = generate_samples(rand_chacha, 0, 100); + let samples = generate_samples(rand_chacha.clone(), 0, 100); + let expected = vec![]; + assert_eq!(samples, expected); + + let samples = generate_samples(rand_chacha, MAX_MODULO_SAMPLES + 1, 100); + let expected = vec![ + 42, 54, 55, 93, 64, 27, 49, 15, 83, 71, 62, 1, 43, 77, 97, 41, 7, 69, 0, 88, 59, 14, + 82, 19, 79, 17, 75, 66, 30, + ] + .into_iter() + .map(Into::into) + .collect_vec(); assert_eq!(samples, expected); } } From ec712151a6168aeebd6f40736cde3539353237a0 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 5 Nov 2023 13:23:22 +0200 Subject: [PATCH 64/89] Fix unittest Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 5e7718edb246..ae0127f6ff23 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -1239,7 +1239,7 @@ mod tests { let samples = generate_samples(rand_chacha, MAX_MODULO_SAMPLES + 1, 100); let expected = vec![ 42, 54, 55, 93, 64, 27, 49, 15, 83, 71, 62, 1, 43, 77, 97, 41, 7, 69, 0, 88, 59, 14, - 82, 19, 79, 17, 75, 66, 30, + 23, 87, 47, 4, 51, 12, 74, 56, 50, 44, 9, 82, 19, 79, 17, 75, 66, 30, ] .into_iter() .map(Into::into) From 552e6fabd18e85d7341cb8b7f8021b8213f94bda Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 5 Nov 2023 14:15:46 +0200 Subject: [PATCH 65/89] Add pr_doc Signed-off-by: Alexandru Gheorghe --- prdoc/pr_1178.prdoc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 prdoc/pr_1178.prdoc diff --git a/prdoc/pr_1178.prdoc b/prdoc/pr_1178.prdoc new file mode 100644 index 000000000000..36c3b05c7a3f --- /dev/null +++ b/prdoc/pr_1178.prdoc @@ -0,0 +1,23 @@ +title: tranche0 assignments in one certificate part1 + +doc: + - audience: Node Operator + description: | + Changed approval-voting, approval-distribution to send all messages tranche0 assignments in one message. + This required: + * A new parachains_db version. + * A new validation protocol to support the new message types. + The new logic will be disabled and will be enabled at a later date after all validators have upgraded. + +migrations: + db: + - name: Parachains database change from v3 to v4. + description: | + Approval-voting column format has been updated with several new fields. All existing data will be automatically + be migrated to the new values. + +crates: + - name: "polkadot" + semver: patch + +host_functions: [] From 2625deffb92cd708d834c9cca6cfa5377efa7830 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 19 Oct 2023 21:47:51 +0300 Subject: [PATCH 66/89] Build a test image with `network-protocol-staging` Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 5c13045706c4..7e673e3fb652 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker - time ROCOCO_EPOCH_DURATION=10 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/ - time ROCOCO_EPOCH_DURATION=100 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-100/ - time ROCOCO_EPOCH_DURATION=600 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-600/ From 43f5529473147285264a656487adf5181dad977b Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 20 Oct 2023 09:54:51 +0300 Subject: [PATCH 67/89] Build a test image with assignments enabled Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index ae0127f6ff23..d398f361eb6b 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -299,7 +299,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) } fn check_assignment_cert( From 590b59e7b5f755394e990e892abbfdd1a53be420 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 5 Nov 2023 15:47:49 +0200 Subject: [PATCH 68/89] Cleanup un-needed structures Signed-off-by: Alexandru Gheorghe --- .../node/core/approval-voting/src/criteria.rs | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index d398f361eb6b..194a7c1781e1 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -130,22 +130,6 @@ fn relay_vrf_modulo_transcript_v2(relay_vrf_story: RelayVRFStory) -> Transcript /// A hard upper bound on num_cores * target_checkers / num_validators const MAX_MODULO_SAMPLES: usize = 40; -use std::convert::AsMut; - -struct BigArray(pub [u8; MAX_MODULO_SAMPLES * 4]); - -impl Default for BigArray { - fn default() -> Self { - BigArray([0u8; MAX_MODULO_SAMPLES * 4]) - } -} - -impl AsMut<[u8]> for BigArray { - fn as_mut(&mut self) -> &mut [u8] { - self.0.as_mut() - } -} - /// Takes the VRF output as input and returns a Vec of cores the validator is assigned /// to as a tranche0 checker. fn relay_vrf_modulo_cores( @@ -194,7 +178,7 @@ fn generate_samples( ); } - let num_samples = min(MAX_MODULO_SAMPLES, min(num_samples, max_cores / 2)); + let num_samples = min(MAX_MODULO_SAMPLES, min(num_samples, max_cores)); let mut random_cores = (0..max_cores as u32).map(|val| val.into()).collect::>(); let (samples, _) = random_cores.partial_shuffle(&mut rand_chacha, num_samples as usize); @@ -1221,7 +1205,7 @@ mod tests { assert_eq!(samples, expected); let samples = generate_samples(rand_chacha.clone(), 6, 7); - let expected = vec![5, 4, 2].into_iter().map(Into::into).collect_vec(); + let expected = vec![0, 3, 6, 5, 4, 2].into_iter().map(Into::into).collect_vec(); assert_eq!(samples, expected); let samples = generate_samples(rand_chacha.clone(), 6, 12); From bd76cf64582acffdd6b9be266be3d8ffcd9e4149 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Sun, 5 Nov 2023 16:28:27 +0200 Subject: [PATCH 69/89] Revert test configurations This reverts commit 43f5529473147285264a656487adf5181dad977b. --- .gitlab/pipeline/build.yml | 2 +- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 7e673e3fb652..5c13045706c4 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -18,7 +18,7 @@ build-linux-stable: # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: - - time cargo build --locked --profile testnet --features pyroscope,network-protocol-staging --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker + - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker - time ROCOCO_EPOCH_DURATION=10 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/ - time ROCOCO_EPOCH_DURATION=100 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-100/ - time ROCOCO_EPOCH_DURATION=600 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-600/ diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 194a7c1781e1..1870f2a6158d 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -283,7 +283,7 @@ impl AssignmentCriteria for RealAssignmentCriteria { config: &Config, leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>, ) -> HashMap { - compute_assignments(keystore, relay_vrf_story, config, leaving_cores, true) + compute_assignments(keystore, relay_vrf_story, config, leaving_cores, false) } fn check_assignment_cert( From 20cb37a8443b29582df994943b90bd0292db11b1 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 6 Nov 2023 09:07:58 +0200 Subject: [PATCH 70/89] Fix upgrade to latest zombienet version Signed-off-by: Alexandru Gheorghe --- .../zombienet_tests/functional/0006-parachains-max-tranche0.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml index 0b75b1a48743..68ac90f5a7e1 100644 --- a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml +++ b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml @@ -10,7 +10,6 @@ bootnode = true [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" -chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" default_command = "polkadot" [relaychain.default_resources] From 311b5732932d982cd49ef8c3f06a8c5555b7941f Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 6 Nov 2023 10:19:21 +0200 Subject: [PATCH 71/89] Set correct log level Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/criteria.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/approval-voting/src/criteria.rs b/polkadot/node/core/approval-voting/src/criteria.rs index 1870f2a6158d..2bb5a151fe23 100644 --- a/polkadot/node/core/approval-voting/src/criteria.rs +++ b/polkadot/node/core/approval-voting/src/criteria.rs @@ -169,7 +169,7 @@ fn generate_samples( } if 2 * num_samples > max_cores { - gum::error!( + gum::debug!( target: LOG_TARGET, n_cores = max_cores, num_samples, From 46f7f4d60a8e892ed37285e68bd1d252378ba0c8 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 6 Nov 2023 18:38:24 +0200 Subject: [PATCH 72/89] Make zombient behave with latest cli Signed-off-by: Alexandru Gheorghe --- .../functional/0007-approval-voting-coalescing.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index 872bf025de97..20dbb3577849 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -4,7 +4,6 @@ timeout = 1000 [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" -chain_spec_command = "polkadot build-spec --chain rococo-local --disable-default-bootnode" [relaychain.genesis.runtime.configuration.config] needed_approvals = 5 From a2e602444662bd0e2443219b438d609da62a995d Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 6 Nov 2023 18:44:24 +0200 Subject: [PATCH 73/89] Fix ocasional no-shows in zombienets ... Because we coalesce approvals they might cover more than one assignments so in the case of the assignments sent randomly we need to make sure we send them all to the peer before we can send the approval Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 274 ++++++++++++++---- 1 file changed, 218 insertions(+), 56 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index f30170b0f116..ed7708b7f0cd 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -418,6 +418,12 @@ impl Knowledge { keys.iter() .all(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) } + + // Tells if all keys are contained by this peer_knowledge + pub fn contains_any(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { + keys.iter() + .any(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) + } } /// Information that has been circulated to and from a peer. @@ -434,6 +440,62 @@ impl PeerKnowledge { self.sent.contains(message, kind) || self.received.contains(message, kind) } + /// Partition a list of assignments into two lists. + /// + /// The the first one contains the list of assignments that we had + /// sent to the peer and the second one the list of assignments that + /// we have no knowledge if the peer has it or not. + fn partition_sent_notknown<'a>( + &self, + assignments: &'a Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, + ) -> ( + Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, + Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, + ) { + let (sent, not_sent): ( + Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, + Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, + ) = assignments.iter().partition(|assignment| { + self.sent.contains( + &MessageSubject( + assignment.0.block_hash, + assignment.1.clone(), + assignment.0.validator, + ), + MessageKind::Assignment, + ) + }); + + let notknown_by_peer = not_sent + .into_iter() + .filter(|(assignment, candidate_bitfield)| { + !self.contains( + &MessageSubject( + assignment.block_hash, + candidate_bitfield.clone(), + assignment.validator, + ), + MessageKind::Assignment, + ) + }) + .collect_vec(); + + (sent, notknown_by_peer) + } + + fn mark_sent(&mut self, assignments: &Vec<(IndirectAssignmentCertV2, CandidateBitfield)>) { + for assignment in assignments { + self.sent.insert( + MessageSubject( + assignment.0.block_hash, + assignment.1.clone(), + assignment.0.validator, + ), + MessageKind::Assignment, + ); + } + } + // Tells if all assignments for a given approval are included in the knowledge of the peer fn contains_assignments_for_approval(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { self.contains_all_keys(&Self::generate_assignments_keys(&approval)) @@ -542,6 +604,40 @@ impl BlockEntry { .iter_ones() .all(|candidate_index| self.candidates.get(candidate_index as usize).is_some()) } + + // Returns all assignments covering the candidates in a given `approval` + pub fn assignments_for_approval( + &self, + approval: &IndirectSignedApprovalVoteV2, + ) -> Result, ApprovalEntryError> { + let mut assignments = Vec::new(); + + if self.candidates.len() < approval.candidate_indices.len() as usize { + return Err(ApprovalEntryError::CandidateIndexOutOfBounds) + } + + // First determine all assignments bitfields that might be covered by this approval + let covered_assignments_bitfields: HashSet = approval + .candidate_indices + .iter_ones() + .filter_map(|candidate_index| { + self.candidates.get(candidate_index).map_or(None, |candidate_entry| { + candidate_entry.assignments.get(&approval.validator).cloned() + }) + }) + .collect(); + + for assignment_bitfield in covered_assignments_bitfields { + if let Some(approval_entry) = + self.approval_entries.get(&(approval.validator, assignment_bitfield.clone())) + { + assignments.push((approval_entry.assignment.clone(), assignment_bitfield.clone())) + } + } + + Ok(assignments) + } + // Saves the given approval in all ApprovalEntries that contain an assignment for any of the // candidates in the approval. // @@ -1425,6 +1521,7 @@ impl State { if !topology.map(|topology| topology.is_validator(&peer)).unwrap_or(false) { continue } + // Note: at this point, we haven't received the message from any peers // other than the source peer, and we just got it, so we haven't sent it // to any peers either. @@ -1715,6 +1812,8 @@ impl State { }, }; + let assignments = entry.assignments_for_approval(&vote).unwrap_or_default(); + // Invariant: to our knowledge, none of the peers except for the `source` know about the // approval. metrics.on_approval_imported(); @@ -1740,7 +1839,7 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || knowledge.sent.contains_all_keys(&assignments_knowledge_keys) + in_topology || knowledge.sent.contains_any(&assignments_knowledge_keys) }; let peers = entry @@ -1754,6 +1853,27 @@ impl State { for peer in peers.iter() { // we already filtered peers above, so this should always be Some if let Some(entry) = entry.known_by.get_mut(&peer.0) { + // A random assignment could have been sent to the peer, so we need to make sure + // we send all the other assignments, before we can send the corresponding approval. + let (sent_to_peer, notknown_by_peer) = entry.partition_sent_notknown(&assignments); + if !notknown_by_peer.is_empty() { + let notknown_by_peer = notknown_by_peer + .into_iter() + .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) + .collect_vec(); + gum::trace!( + target: LOG_TARGET, + ?block_hash, + ?peer, + missing = notknown_by_peer.len(), + part1 = sent_to_peer.len(), + "Sending missing assignments", + ); + + entry.mark_sent(¬known_by_peer); + send_assignments_batched(ctx.sender(), notknown_by_peer, &[(peer.0, peer.1)]).await; + } + entry.sent.insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } } @@ -1851,75 +1971,99 @@ impl State { if entry.known_by.contains_key(&peer_id) { break } - - let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let topology = topologies.get_topology(entry.session); - - let mut approvals_to_send_this_block = HashMap::new(); - // We want to iterate the `approval_entries` of the block entry as these contain all - // assignments that also link all approval votes. - for approval_entry in entry.approval_entries.values_mut() { - // Propagate the message to all peers in the required routing set OR - // randomly sample peers. - { - let required_routing = approval_entry.routing_info().required_routing; - let random_routing = &mut approval_entry.routing_info_mut().random_routing; - let rng = &mut *rng; - let mut peer_filter = move |peer_id| { - let in_topology = topology.as_ref().map_or(false, |t| { - t.local_grid_neighbors().route_to_peer(required_routing, peer_id) - }); - in_topology || { - if !topology - .map(|topology| topology.is_validator(peer_id)) - .unwrap_or(false) - { - return false - } - - let route_random = random_routing.sample(total_peers, rng); - if route_random { - random_routing.inc_sent(); + let approvals_to_send_this_block = { + let peer_knowledge = entry.known_by.entry(peer_id).or_default(); + let topology = topologies.get_topology(entry.session); + + let mut approvals_to_send_this_block = HashMap::new(); + // We want to iterate the `approval_entries` of the block entry as these contain + // all assignments that also link all approval votes. + for approval_entry in entry.approval_entries.values_mut() { + // Propagate the message to all peers in the required routing set OR + // randomly sample peers. + { + let required_routing = approval_entry.routing_info().required_routing; + let random_routing = + &mut approval_entry.routing_info_mut().random_routing; + let rng = &mut *rng; + let mut peer_filter = move |peer_id| { + let in_topology = topology.as_ref().map_or(false, |t| { + t.local_grid_neighbors() + .route_to_peer(required_routing, peer_id) + }); + in_topology || { + if !topology + .map(|topology| topology.is_validator(peer_id)) + .unwrap_or(false) + { + return false + } + + let route_random = random_routing.sample(total_peers, rng); + if route_random { + random_routing.inc_sent(); + } + + route_random } + }; - route_random + if !peer_filter(&peer_id) { + continue } - }; - - if !peer_filter(&peer_id) { - continue } - } - let assignment_message = approval_entry.assignment(); - let approval_messages = approval_entry.approvals(); - let (assignment_knowledge, message_kind) = - approval_entry.create_assignment_knowledge(block); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(block); - // Only send stuff a peer doesn't know in the context of a relay chain block. - if !peer_knowledge.contains(&assignment_knowledge, message_kind) { - peer_knowledge.sent.insert(assignment_knowledge, message_kind); - assignments_to_send.push(assignment_message); - } + // Only send stuff a peer doesn't know in the context of a relay chain + // block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge, message_kind); + assignments_to_send.push(assignment_message); + } - // Filter approval votes. - for approval_message in approval_messages { - let approval_knowledge = - PeerKnowledge::generate_approval_key(&approval_message); + // Filter approval votes. + for approval_message in approval_messages { + let approval_knowledge = + PeerKnowledge::generate_approval_key(&approval_message); - if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) { - if !approvals_to_send_this_block.contains_key(&approval_knowledge.0) { - approvals_to_send_this_block - .insert(approval_knowledge.0.clone(), approval_message); + if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) + { + if !approvals_to_send_this_block.contains_key(&approval_knowledge.0) + { + approvals_to_send_this_block + .insert(approval_knowledge.0.clone(), approval_message); + } } } } - } + approvals_to_send_this_block + }; // An approval can span multiple assignments/ApprovalEntries, so after we processed // all of the assignments decide which of the approvals we can safely send, because // all of assignments are already sent or about to be sent. for (approval_key, approval) in approvals_to_send_this_block { + let assignments = entry.assignments_for_approval(&approval).unwrap_or_default(); + let peer_knowledge = entry.known_by.entry(peer_id).or_default(); + let (sent_to_peer, notknown_by_peer) = peer_knowledge.partition_sent_notknown(&assignments); + if !sent_to_peer.is_empty() { + let notknown_by_peer = notknown_by_peer + .into_iter() + .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) + .collect_vec(); + gum::trace!( + target: LOG_TARGET, + ?notknown_by_peer, + part1 = sent_to_peer.len(), + "Sending missing assignment unify_with_peer", + ); + peer_knowledge.mark_sent(¬known_by_peer); + assignments_to_send.extend(notknown_by_peer); + } if peer_knowledge.contains_assignments_for_approval(&approval) { approvals_to_send.push(approval); peer_knowledge.sent.insert(approval_key, MessageKind::Approval); @@ -2243,8 +2387,26 @@ async fn adjust_required_routing_and_propagate Date: Tue, 7 Nov 2023 10:31:35 +0200 Subject: [PATCH 74/89] Fixup comment Signed-off-by: Alexandru Gheorghe --- polkadot/primitives/src/v6/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/polkadot/primitives/src/v6/mod.rs b/polkadot/primitives/src/v6/mod.rs index 8f11092b8455..fe571236109e 100644 --- a/polkadot/primitives/src/v6/mod.rs +++ b/polkadot/primitives/src/v6/mod.rs @@ -1367,8 +1367,9 @@ pub enum ValidDisputeStatementKind { #[codec(index = 3)] ApprovalChecking, /// An approval vote from the new version. - /// TODO: Fixme this probably means we can't create this version - /// untill all nodes have been updated to support it. + /// We can't create this version untill all nodes + /// have been updated to support it and max_approval_coalesce_count + /// is set to more than 1. #[codec(index = 4)] ApprovalCheckingMultipleCandidates(Vec), } From 624bb5f9e66eed9fad5d9319285580b6ccc226c6 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 7 Nov 2023 11:15:14 +0200 Subject: [PATCH 75/89] Fix genesis error in zombienet Signed-off-by: Alexandru Gheorghe --- .../functional/0007-approval-voting-coalescing.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index 20dbb3577849..f84a87f918bd 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -5,11 +5,11 @@ timeout = 1000 default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" -[relaychain.genesis.runtime.configuration.config] +[relaychain.genesis.runtimeGenesis.patch.configuration.config] needed_approvals = 5 relay_vrf_modulo_samples = 6 -[relaychain.genesis.runtime.configuration.config.approval_voting_params] +[relaychain.genesis.runtimeGenesis.patch.configuration.config.approval_voting_params] max_approval_coalesce_count = 5 [relaychain.default_resources] From b0faa095c88f2084fbacf0ce645c29cadeb872a4 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Wed, 8 Nov 2023 12:23:33 +0200 Subject: [PATCH 76/89] Minor fixes Signed-off-by: Alexandru Gheorghe --- .../src/approval_db/v1/tests.rs | 4 ---- .../src/approval_db/v2/tests.rs | 7 +------ .../src/approval_db/v3/migration_helpers.rs | 2 +- .../approval-voting/src/approval_db/v3/mod.rs | 6 ++---- .../src/approval_db/v3/tests.rs | 7 +------ polkadot/node/core/approval-voting/src/lib.rs | 18 ++++++++++++++---- .../approval-voting/src/persisted_entries.rs | 10 +++++----- .../node/core/approval-voting/src/tests.rs | 2 +- .../network/approval-distribution/src/lib.rs | 1 + polkadot/runtime/parachains/src/disputes.rs | 2 ++ .../0007-approval-voting-coalescing.toml | 4 ++-- 11 files changed, 30 insertions(+), 33 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/approval_db/v1/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v1/tests.rs index 07d8242b772e..b979cb7ef45f 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v1/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v1/tests.rs @@ -40,10 +40,6 @@ fn make_db() -> (DbBackend, Arc) { (DbBackend::new(db_writer.clone(), TEST_CONFIG), db_writer) } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - fn make_block_entry( block_hash: Hash, parent_hash: Hash, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs index 1121660fe658..3dbda3639d28 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -18,7 +18,7 @@ use crate::{ approval_db::{ - common::{DbBackend, StoredBlockRange, *}, + common::{DbBackend, StoredBlockRange, *, migration_helpers::make_bitvec}, v2::*, v3::{load_block_entry_v2, load_candidate_entry_v2}, }, @@ -29,7 +29,6 @@ use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, }; -use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::Id as ParaId; use sp_consensus_slots::Slot; @@ -70,10 +69,6 @@ fn make_block_entry( } } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { let mut c = dummy_candidate_receipt(dummy_hash()); diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs index 3ffb66015215..ad5e89ef3de8 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/migration_helpers.rs @@ -116,7 +116,7 @@ pub fn v1_to_latest_sanity_check( Ok(()) } -// Fills the db with dummy data in v1 scheme. +// Fills the db with dummy data in v2 scheme. pub fn v2_fill_test_data( db: Arc, config: Config, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs index 031a51edd64b..3e4f43021952 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/mod.rs @@ -16,8 +16,7 @@ //! Version 3 of the DB schema. //! -//! Version 3 modifies the our_approval format of `ApprovalEntry` -//! and adds a new field `pending_signatures` for `BlockEntry` +//! Version 3 modifies the `our_approval` format of `ApprovalEntry` //! and adds a new field `pending_signatures` for `BlockEntry` use parity_scale_codec::{Decode, Encode}; @@ -107,8 +106,7 @@ pub struct BlockEntry { } #[derive(Encode, Decode, Debug, Clone, PartialEq)] - -/// Context needed for creating an approval signature for a given candidate. +/// Context needed for creating an approval signature for a given candidate. pub struct CandidateSigningContext { /// The candidate hash, to be included in the signature. pub candidate_hash: CandidateHash, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs index 7ff6433b5365..29b3373e3f25 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs @@ -18,7 +18,7 @@ use crate::{ approval_db::{ - common::{DbBackend, StoredBlockRange, *}, + common::{DbBackend, StoredBlockRange, *, migration_helpers::make_bitvec}, v3::*, }, backend::{Backend, OverlayedBackend}, @@ -28,7 +28,6 @@ use polkadot_primitives::{ BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, }; -use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use polkadot_node_subsystem_util::database::Database; use polkadot_primitives::Id as ParaId; use sp_consensus_slots::Slot; @@ -70,10 +69,6 @@ fn make_block_entry( } } -fn make_bitvec(len: usize) -> BitVec { - bitvec::bitvec![u8, BitOrderLsb0; 0; len] -} - fn make_candidate(para_id: ParaId, relay_parent: Hash) -> CandidateReceipt { let mut c = dummy_candidate_receipt(dummy_hash()); diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index f6d843a908e7..4909a4d6ad9a 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -1018,6 +1018,8 @@ where (block_hash, validator_index) = delayed_approvals_timers.select_next_some() => { gum::debug!( target: LOG_TARGET, + ?block_hash, + ?validator_index, "Sign approval for multiple candidates", ); @@ -1740,7 +1742,7 @@ async fn get_approval_signatures_for_candidate( Some(Ok(votes)) => { let votes = votes .into_iter() - .map(|(validator_index, (hash, signed_candidates_indices, signature))| { + .filter_map(|(validator_index, (hash, signed_candidates_indices, signature))| { let candidates_hashes = candidate_indices_to_candidate_hashes.get(&hash); if candidates_hashes.is_none() { @@ -1751,6 +1753,8 @@ async fn get_approval_signatures_for_candidate( ); } + let num_signed_candidates = signed_candidates_indices.len(); + let signed_candidates_hashes: Vec = signed_candidates_indices .into_iter() @@ -1771,7 +1775,15 @@ async fn get_approval_signatures_for_candidate( }) }) .collect(); - (validator_index, (signed_candidates_hashes, signature)) + if num_signed_candidates == signed_candidates_hashes.len() { + Some((validator_index, (signed_candidates_hashes, signature))) + } else { + gum::warn!( + target: LOG_TARGET, + "Possible bug! Could not find all hashes for candidates coming from approval-distribution" + ); + None + } }) .collect(); send_votes(votes) @@ -3476,8 +3488,6 @@ async fn maybe_create_signature( .collect::>>>()?; for mut candidate_entry in candidate_entries { - // let mut candidate_entry = candidate_entry - // .expect("Candidate was scheduled to be signed entry in db should exist; qed"); let approval_entry = candidate_entry.as_mut().and_then(|candidate_entry| { candidate_entry.approval_entry_mut(&block_entry.block_hash()) }); diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index ac3b704c31c2..39c1fffc2ee7 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -597,13 +597,12 @@ impl BlockEntry { } /// Candidate indices for candidates pending signature - fn candidate_indices_pending_signature(&self) -> CandidateBitfield { + fn candidate_indices_pending_signature(&self) -> Option { self.candidates_pending_signature .keys() .map(|val| *val) .collect_vec() - .try_into() - .expect("Fails only of array empty, it can't be, qed") + .try_into().ok() } /// Returns a list of candidates hashes that need need signature created at the current tick: @@ -629,10 +628,11 @@ impl BlockEntry { self.num_candidates_pending_signature() >= max_approval_coalesce_count as usize { ( + self.candidate_indices_pending_signature().and_then(|candidate_indices| Some(( self.candidate_hashes_pending_signature(), - self.candidate_indices_pending_signature(), - )), + candidate_indices, + ))), Some(sign_no_later_than_tick), ) } else { diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index a27315d416e3..941c2ec5a32b 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -2028,7 +2028,7 @@ fn test_signing_a_single_candidate_is_backwards_compatible() { .map(|candidate_descriptor| candidate_descriptor.hash()) .collect_vec(); - let first_descriptor = candidate_descriptors.first().expect("TODO"); + let first_descriptor = candidate_descriptors.first().unwrap(); let candidate_hash = first_descriptor.hash(); diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index cf70d253d48b..2ebf050849f1 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -483,6 +483,7 @@ impl PeerKnowledge { (sent, notknown_by_peer) } + /// Marks a list of assignments as sent to the peer fn mark_sent(&mut self, assignments: &Vec<(IndirectAssignmentCertV2, CandidateBitfield)>) { for assignment in assignments { self.sent.insert( diff --git a/polkadot/runtime/parachains/src/disputes.rs b/polkadot/runtime/parachains/src/disputes.rs index fce5dc24d28d..ca275cf6442c 100644 --- a/polkadot/runtime/parachains/src/disputes.rs +++ b/polkadot/runtime/parachains/src/disputes.rs @@ -1017,6 +1017,8 @@ impl Pallet { set.session, statement, signature, + // This is here to prevent malicious nodes of generating `ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates` + // before that is enabled, via setting `max_approval_coalesce_count` in the parachain host config. config.approval_voting_params.max_approval_coalesce_count > 1, ) { log::warn!("Failed to check dispute signature"); diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml index f84a87f918bd..19c7015403d7 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml @@ -6,7 +6,7 @@ default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" [relaychain.genesis.runtimeGenesis.patch.configuration.config] - needed_approvals = 5 + needed_approvals = 4 relay_vrf_modulo_samples = 6 [relaychain.genesis.runtimeGenesis.patch.configuration.config.approval_voting_params] @@ -19,7 +19,7 @@ requests = { memory = "2G", cpu = "1" } [[relaychain.node_groups]] name = "alice" args = [ "-lparachain=trace,runtime=debug" ] - count = 21 + count = 13 [[parachains]] id = 2000 From 5e781278227c62354a63c54d2b0a625cfed6c760 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 7 Dec 2023 15:01:00 +0200 Subject: [PATCH 77/89] Revert a2e602444662bd0e2443219b438d609da62a995d ... it is actually not needed and gets us a preformance penalty which I discovered while working on: https://github.com/paritytech/polkadot-sdk/pull/2621 Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 357 ++++-------------- 1 file changed, 82 insertions(+), 275 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 2ebf050849f1..edba3e1af2da 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -114,6 +114,14 @@ struct ApprovalRouting { required_routing: RequiredRouting, local: bool, random_routing: RandomRouting, + peers_randomly_routed: Vec, +} + +impl ApprovalRouting { + fn mark_randomly_sent(&mut self, peer: PeerId) { + self.random_routing.inc_sent(); + self.peers_randomly_routed.push(peer); + } } // This struct is responsible for tracking the full state of an assignment and grid routing @@ -412,18 +420,6 @@ impl Knowledge { } success } - - // Tells if all keys are contained by this peer_knowledge - pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { - keys.iter() - .all(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) - } - - // Tells if all keys are contained by this peer_knowledge - pub fn contains_any(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { - keys.iter() - .any(|assignment_key| self.contains(&assignment_key.0, assignment_key.1)) - } } /// Information that has been circulated to and from a peer. @@ -440,73 +436,6 @@ impl PeerKnowledge { self.sent.contains(message, kind) || self.received.contains(message, kind) } - /// Partition a list of assignments into two lists. - /// - /// The the first one contains the list of assignments that we had - /// sent to the peer and the second one the list of assignments that - /// we have no knowledge if the peer has it or not. - fn partition_sent_notknown<'a>( - &self, - assignments: &'a Vec<(IndirectAssignmentCertV2, CandidateBitfield)>, - ) -> ( - Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, - Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, - ) { - let (sent, not_sent): ( - Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, - Vec<&'a (IndirectAssignmentCertV2, CandidateBitfield)>, - ) = assignments.iter().partition(|assignment| { - self.sent.contains( - &MessageSubject( - assignment.0.block_hash, - assignment.1.clone(), - assignment.0.validator, - ), - MessageKind::Assignment, - ) - }); - - let notknown_by_peer = not_sent - .into_iter() - .filter(|(assignment, candidate_bitfield)| { - !self.contains( - &MessageSubject( - assignment.block_hash, - candidate_bitfield.clone(), - assignment.validator, - ), - MessageKind::Assignment, - ) - }) - .collect_vec(); - - (sent, notknown_by_peer) - } - - /// Marks a list of assignments as sent to the peer - fn mark_sent(&mut self, assignments: &Vec<(IndirectAssignmentCertV2, CandidateBitfield)>) { - for assignment in assignments { - self.sent.insert( - MessageSubject( - assignment.0.block_hash, - assignment.1.clone(), - assignment.0.validator, - ), - MessageKind::Assignment, - ); - } - } - - // Tells if all assignments for a given approval are included in the knowledge of the peer - fn contains_assignments_for_approval(&self, approval: &IndirectSignedApprovalVoteV2) -> bool { - self.contains_all_keys(&Self::generate_assignments_keys(&approval)) - } - - // Tells if all keys are contained by this peer_knowledge - pub fn contains_all_keys(&self, keys: &Vec<(MessageSubject, MessageKind)>) -> bool { - self.sent.contains_all_keys(keys) || self.received.contains_all_keys(keys) - } - // Generate the knowledge keys for querying if all assignments of an approval are known // by this peer. fn generate_assignments_keys( @@ -606,48 +535,17 @@ impl BlockEntry { .all(|candidate_index| self.candidates.get(candidate_index as usize).is_some()) } - // Returns all assignments covering the candidates in a given `approval` - pub fn assignments_for_approval( - &self, - approval: &IndirectSignedApprovalVoteV2, - ) -> Result, ApprovalEntryError> { - let mut assignments = Vec::new(); - - if self.candidates.len() < approval.candidate_indices.len() as usize { - return Err(ApprovalEntryError::CandidateIndexOutOfBounds) - } - - // First determine all assignments bitfields that might be covered by this approval - let covered_assignments_bitfields: HashSet = approval - .candidate_indices - .iter_ones() - .filter_map(|candidate_index| { - self.candidates.get(candidate_index).map_or(None, |candidate_entry| { - candidate_entry.assignments.get(&approval.validator).cloned() - }) - }) - .collect(); - - for assignment_bitfield in covered_assignments_bitfields { - if let Some(approval_entry) = - self.approval_entries.get(&(approval.validator, assignment_bitfield.clone())) - { - assignments.push((approval_entry.assignment.clone(), assignment_bitfield.clone())) - } - } - - Ok(assignments) - } - // Saves the given approval in all ApprovalEntries that contain an assignment for any of the // candidates in the approval. // - // Returns the required routing needed for this approval. + // Returns the required routing needed for this approval and the lit of random peers the + // covering assignments were sent. pub fn note_approval( &mut self, approval: IndirectSignedApprovalVoteV2, - ) -> Result { + ) -> Result<(RequiredRouting, HashSet), ApprovalEntryError> { let mut required_routing = None; + let mut peers_randomly_routed_to = HashSet::new(); if self.candidates.len() < approval.candidate_indices.len() as usize { return Err(ApprovalEntryError::CandidateIndexOutOfBounds) @@ -670,6 +568,8 @@ impl BlockEntry { self.approval_entries.get_mut(&(approval.validator, assignment_bitfield)) { approval_entry.note_approval(approval.clone())?; + peers_randomly_routed_to + .extend(approval_entry.routing_info().peers_randomly_routed.iter()); if let Some(required_routing) = required_routing { if required_routing != approval_entry.routing_info().required_routing { @@ -688,7 +588,7 @@ impl BlockEntry { } if let Some(required_routing) = required_routing { - Ok(required_routing) + Ok((required_routing, peers_randomly_routed_to)) } else { Err(ApprovalEntryError::UnknownAssignment) } @@ -1489,7 +1389,12 @@ impl State { let approval_entry = entry.insert_approval_entry(ApprovalEntry::new( assignment.clone(), claimed_candidate_indices.clone(), - ApprovalRouting { required_routing, local, random_routing: Default::default() }, + ApprovalRouting { + required_routing, + local, + random_routing: Default::default(), + peers_randomly_routed: Default::default(), + }, )); // Dispatch the message to all peers in the routing set which @@ -1530,7 +1435,7 @@ impl State { approval_entry.routing_info().random_routing.sample(n_peers_total, rng); if route_random { - approval_entry.routing_info_mut().random_routing.inc_sent(); + approval_entry.routing_info_mut().mark_randomly_sent(peer); peers.push(peer); } } @@ -1797,7 +1702,7 @@ impl State { } } - let required_routing = match entry.note_approval(vote.clone()) { + let (required_routing, peers_randomly_routed_to) = match entry.note_approval(vote.clone()) { Ok(required_routing) => required_routing, Err(err) => { gum::warn!( @@ -1813,8 +1718,6 @@ impl State { }, }; - let assignments = entry.assignments_for_approval(&vote).unwrap_or_default(); - // Invariant: to our knowledge, none of the peers except for the `source` know about the // approval. metrics.on_approval_imported(); @@ -1824,7 +1727,7 @@ impl State { let topology = self.topologies.get_topology(entry.session); let source_peer = source.peer_id(); - let peer_filter = move |peer, knowledge: &PeerKnowledge| { + let peer_filter = move |peer| { if Some(peer) == source_peer.as_ref() { return false } @@ -1840,13 +1743,13 @@ impl State { // 3. Any randomly selected peers have been sent the assignment already. let in_topology = topology .map_or(false, |t| t.local_grid_neighbors().route_to_peer(required_routing, peer)); - in_topology || knowledge.sent.contains_any(&assignments_knowledge_keys) + in_topology || peers_randomly_routed_to.contains(peer) }; let peers = entry .known_by .iter() - .filter(|(p, k)| peer_filter(p, k)) + .filter(|(p, _)| peer_filter(p)) .filter_map(|(p, _)| self.peer_views.get(p).map(|entry| (*p, entry.version))) .collect::>(); @@ -1854,28 +1757,6 @@ impl State { for peer in peers.iter() { // we already filtered peers above, so this should always be Some if let Some(entry) = entry.known_by.get_mut(&peer.0) { - // A random assignment could have been sent to the peer, so we need to make sure - // we send all the other assignments, before we can send the corresponding approval. - let (sent_to_peer, notknown_by_peer) = entry.partition_sent_notknown(&assignments); - if !notknown_by_peer.is_empty() { - let notknown_by_peer = notknown_by_peer - .into_iter() - .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) - .collect_vec(); - gum::trace!( - target: LOG_TARGET, - ?block_hash, - ?peer, - missing = notknown_by_peer.len(), - part1 = sent_to_peer.len(), - "Sending missing assignments", - ); - - entry.mark_sent(¬known_by_peer); - send_assignments_batched(ctx.sender(), notknown_by_peer, &[(peer.0, peer.1)]) - .await; - } - entry.sent.insert(approval_knwowledge_key.0.clone(), approval_knwowledge_key.1); } } @@ -1973,105 +1854,70 @@ impl State { if entry.known_by.contains_key(&peer_id) { break } - let approvals_to_send_this_block = { - let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let topology = topologies.get_topology(entry.session); - - let mut approvals_to_send_this_block = HashMap::new(); - // We want to iterate the `approval_entries` of the block entry as these contain - // all assignments that also link all approval votes. - for approval_entry in entry.approval_entries.values_mut() { - // Propagate the message to all peers in the required routing set OR - // randomly sample peers. - { - let required_routing = approval_entry.routing_info().required_routing; - let random_routing = - &mut approval_entry.routing_info_mut().random_routing; - let rng = &mut *rng; - let mut peer_filter = move |peer_id| { - let in_topology = topology.as_ref().map_or(false, |t| { - t.local_grid_neighbors() - .route_to_peer(required_routing, peer_id) - }); - in_topology || { - if !topology - .map(|topology| topology.is_validator(peer_id)) - .unwrap_or(false) - { - return false - } - - let route_random = random_routing.sample(total_peers, rng); - if route_random { - random_routing.inc_sent(); - } - - route_random - } - }; - - if !peer_filter(&peer_id) { - continue - } - } - - let assignment_message = approval_entry.assignment(); - let approval_messages = approval_entry.approvals(); - let (assignment_knowledge, message_kind) = - approval_entry.create_assignment_knowledge(block); - - // Only send stuff a peer doesn't know in the context of a relay chain - // block. - if !peer_knowledge.contains(&assignment_knowledge, message_kind) { - peer_knowledge.sent.insert(assignment_knowledge, message_kind); - assignments_to_send.push(assignment_message); - } - // Filter approval votes. - for approval_message in approval_messages { - let approval_knowledge = - PeerKnowledge::generate_approval_key(&approval_message); + let peer_knowledge = entry.known_by.entry(peer_id).or_default(); + let topology = topologies.get_topology(entry.session); - if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) - { - if !approvals_to_send_this_block.contains_key(&approval_knowledge.0) + // We want to iterate the `approval_entries` of the block entry as these contain + // all assignments that also link all approval votes. + for approval_entry in entry.approval_entries.values_mut() { + // Propagate the message to all peers in the required routing set OR + // randomly sample peers. + { + let required_routing = approval_entry.routing_info().required_routing; + let routing_info = &mut approval_entry.routing_info_mut(); + let rng = &mut *rng; + let mut peer_filter = move |peer_id| { + let in_topology = topology.as_ref().map_or(false, |t| { + t.local_grid_neighbors().route_to_peer(required_routing, peer_id) + }); + in_topology || { + if !topology + .map(|topology| topology.is_validator(peer_id)) + .unwrap_or(false) { - approvals_to_send_this_block - .insert(approval_knowledge.0.clone(), approval_message); + return false + } + + let route_random = + routing_info.random_routing.sample(total_peers, rng); + if route_random { + routing_info.mark_randomly_sent(*peer_id); } + + route_random } + }; + + if !peer_filter(&peer_id) { + continue } } - approvals_to_send_this_block - }; - // An approval can span multiple assignments/ApprovalEntries, so after we processed - // all of the assignments decide which of the approvals we can safely send, because - // all of assignments are already sent or about to be sent. - for (approval_key, approval) in approvals_to_send_this_block { - let assignments = entry.assignments_for_approval(&approval).unwrap_or_default(); - let peer_knowledge = entry.known_by.entry(peer_id).or_default(); - let (sent_to_peer, notknown_by_peer) = - peer_knowledge.partition_sent_notknown(&assignments); - if !sent_to_peer.is_empty() { - let notknown_by_peer = notknown_by_peer - .into_iter() - .map(|(assignment, bitfield)| (assignment.clone(), bitfield.clone())) - .collect_vec(); - gum::trace!( - target: LOG_TARGET, - ?notknown_by_peer, - part1 = sent_to_peer.len(), - "Sending missing assignment unify_with_peer", - ); - peer_knowledge.mark_sent(¬known_by_peer); - assignments_to_send.extend(notknown_by_peer); + let assignment_message = approval_entry.assignment(); + let approval_messages = approval_entry.approvals(); + let (assignment_knowledge, message_kind) = + approval_entry.create_assignment_knowledge(block); + + // Only send stuff a peer doesn't know in the context of a relay chain + // block. + if !peer_knowledge.contains(&assignment_knowledge, message_kind) { + peer_knowledge.sent.insert(assignment_knowledge, message_kind); + assignments_to_send.push(assignment_message); } - if peer_knowledge.contains_assignments_for_approval(&approval) { - approvals_to_send.push(approval); - peer_knowledge.sent.insert(approval_key, MessageKind::Approval); + + // Filter approval votes. + for approval_message in approval_messages { + let approval_knowledge = + PeerKnowledge::generate_approval_key(&approval_message); + + if !peer_knowledge.contains(&approval_knowledge.0, approval_knowledge.1) { + approvals_to_send.push(approval_message); + peer_knowledge.sent.insert(approval_knowledge.0, approval_knowledge.1); + } } } + block = entry.parent_hash; } } @@ -2334,8 +2180,6 @@ async fn adjust_required_routing_and_propagate continue, }; - let mut approvals_to_send_for_this_block = HashMap::new(); - // We just need to iterate the `approval_entries` of the block entry as these contain all // assignments that also link all approval votes. for approval_entry in block_entry.approval_entries.values_mut() { @@ -2372,8 +2216,6 @@ async fn adjust_required_routing_and_propagate Date: Thu, 7 Dec 2023 17:48:16 +0200 Subject: [PATCH 78/89] Address review feedback Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 29 ++++++++++--------- .../approval-voting/src/persisted_entries.rs | 13 ++++----- polkadot/primitives/src/v6/mod.rs | 2 +- polkadot/primitives/src/vstaging/mod.rs | 6 ++++ .../src/protocol-approval.md | 6 ++-- .../0007-approval-voting-coalescing.zndsl | 2 +- 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index f0356e6ded35..af76b576d7ca 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -174,7 +174,7 @@ struct MetricsInner { approved_by_one_third: prometheus::Counter, wakeups_triggered_total: prometheus::Counter, coalesced_approvals_buckets: prometheus::Histogram, - coalesced_approvals_waiting_times: prometheus::Histogram, + coalesced_approvals_delay: prometheus::Histogram, candidate_approval_time_ticks: prometheus::Histogram, block_approval_time_ticks: prometheus::Histogram, time_db_transaction: prometheus::Histogram, @@ -212,7 +212,7 @@ impl Metrics { fn on_delayed_approval(&self, delayed_ticks: u64) { if let Some(metrics) = &self.0 { - metrics.coalesced_approvals_waiting_times.observe(delayed_ticks as f64) + metrics.coalesced_approvals_delay.observe(delayed_ticks as f64) } } @@ -373,10 +373,10 @@ impl metrics::Metrics for Metrics { )?, registry, )?, - coalesced_approvals_waiting_times: prometheus::register( + coalesced_approvals_delay: prometheus::register( prometheus::Histogram::with_opts( prometheus::HistogramOpts::new( - "polkadot_parachain_approvals_coalesced_approvals_waiting_times", + "polkadot_parachain_approvals_coalescing_delay", "Number of ticks we delay the sending of a candidate approval", ).buckets(vec![1.1, 2.1, 3.1, 4.1, 6.1, 8.1, 12.1, 20.1, 32.1]), )?, @@ -840,7 +840,7 @@ impl State { ctx: &mut Context, session_index: SessionIndex, block_hash: Hash, - ) -> ApprovalVotingParams { + ) -> Option { let (s_tx, s_rx) = oneshot::channel(); ctx.send_message(RuntimeApiMessage::Request( @@ -857,23 +857,23 @@ impl State { session = ?session_index, "Using the following subsystem params" ); - params + Some(params) }, Ok(Err(err)) => { gum::debug!( target: LOG_TARGET, ?err, - "Could not request approval voting params from runtime using defaults" + "Could not request approval voting params from runtime" ); - ApprovalVotingParams { max_approval_coalesce_count: 1 } + None }, Err(err) => { gum::debug!( target: LOG_TARGET, ?err, - "Could not request approval voting params from runtime using defaults" + "Could not request approval voting params from runtime" ); - ApprovalVotingParams { max_approval_coalesce_count: 1 } + None }, } } @@ -3411,7 +3411,8 @@ async fn maybe_create_signature( let approval_params = state .get_approval_voting_params_or_default(ctx, block_entry.session(), block_hash) - .await; + .await + .unwrap_or_default(); gum::trace!( target: LOG_TARGET, @@ -3464,7 +3465,7 @@ async fn maybe_create_signature( let signature = match sign_approval( &state.keystore, &validator_pubkey, - candidates_hashes.clone(), + &candidates_hashes, block_entry.session(), ) { Some(sig) => sig, @@ -3534,12 +3535,12 @@ async fn maybe_create_signature( fn sign_approval( keystore: &LocalKeystore, public: &ValidatorId, - candidate_hashes: Vec, + candidate_hashes: &[CandidateHash], session_index: SessionIndex, ) -> Option { let key = keystore.key_pair::(public).ok().flatten()?; - let payload = ApprovalVoteMultipleCandidates(&candidate_hashes).signing_payload(session_index); + let payload = ApprovalVoteMultipleCandidates(candidate_hashes).signing_payload(session_index); Some(key.sign(&payload[..])) } diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index 39c1fffc2ee7..ef47bdb2213a 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -602,7 +602,8 @@ impl BlockEntry { .keys() .map(|val| *val) .collect_vec() - .try_into().ok() + .try_into() + .ok() } /// Returns a list of candidates hashes that need need signature created at the current tick: @@ -628,11 +629,9 @@ impl BlockEntry { self.num_candidates_pending_signature() >= max_approval_coalesce_count as usize { ( - self.candidate_indices_pending_signature().and_then(|candidate_indices| - Some(( - self.candidate_hashes_pending_signature(), - candidate_indices, - ))), + self.candidate_indices_pending_signature().and_then(|candidate_indices| { + Some((self.candidate_hashes_pending_signature(), candidate_indices)) + }), Some(sign_no_later_than_tick), ) } else { @@ -648,7 +647,7 @@ impl BlockEntry { } } - /// Signals the approval was issued for the candidates pending signature + /// Clears the candidates pending signature because the approval was issued. pub fn issued_approval(&mut self) { self.candidates_pending_signature.clear(); } diff --git a/polkadot/primitives/src/v6/mod.rs b/polkadot/primitives/src/v6/mod.rs index 6b3920f48ff3..c3a947644fff 100644 --- a/polkadot/primitives/src/v6/mod.rs +++ b/polkadot/primitives/src/v6/mod.rs @@ -1072,7 +1072,7 @@ impl ApprovalVote { /// A vote of approval for multiple candidates. #[derive(Clone, RuntimeDebug)] -pub struct ApprovalVoteMultipleCandidates<'a>(pub &'a Vec); +pub struct ApprovalVoteMultipleCandidates<'a>(pub &'a [CandidateHash]); impl<'a> ApprovalVoteMultipleCandidates<'a> { /// Yields the signing payload for this approval vote. diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 1fe72eaa2b8f..7d213b8e7a73 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -44,6 +44,12 @@ pub struct ApprovalVotingParams { pub max_approval_coalesce_count: u32, } +impl Default for ApprovalVotingParams { + fn default() -> Self { + Self { max_approval_coalesce_count: 1 } + } +} + use bitvec::vec::BitVec; /// Bit indices in the `HostConfiguration.node_features` that correspond to different node features. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index f3009f5fb44a..ebe4894f0b3e 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -301,8 +301,10 @@ To reduce the necessary network bandwidth and cpu time when a validator has more doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we delay sending it until one of three things happen: -- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we are ready to vote for. -- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since the we were ready to approve the candidate. +- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we have already checked and we are + ready to sign approval for. +- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since checking oldest candidate and we were ready to sign + and send the approval message. - We are already in the last third of the no-show period in order to avoid creating accidental no-shows, which in turn might trigger other assignments. diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl index 07ef30d546a1..43b2c5e13a80 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl +++ b/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl @@ -29,4 +29,4 @@ alice: reports substrate_block_height{status="finalized"} is at least 30 within alice: reports polkadot_parachain_approval_checking_finality_lag < 3 -alice: reports polkadot_parachain_approvals_no_shows_total < 1 within 10 seconds +alice: reports polkadot_parachain_approvals_no_shows_total < 3 within 10 seconds From 1913b63e6dca44b496b41c53d54dfabe1bd23780 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 7 Dec 2023 18:24:00 +0200 Subject: [PATCH 79/89] protocol: rename vstaging into v3 Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 51 +++--- .../approval-distribution/src/tests.rs | 146 +++++++++--------- .../network/bitfield-distribution/src/lib.rs | 18 +-- polkadot/node/network/bridge/src/network.rs | 12 +- polkadot/node/network/bridge/src/rx/mod.rs | 97 ++++++------ polkadot/node/network/bridge/src/rx/tests.rs | 6 +- polkadot/node/network/bridge/src/tx/mod.rs | 10 +- .../src/collator_side/mod.rs | 6 +- .../src/validator_side/mod.rs | 6 +- .../node/network/gossip-support/src/lib.rs | 2 +- polkadot/node/network/protocol/src/lib.rs | 54 ++++--- .../node/network/protocol/src/peer_set.rs | 11 +- .../src/legacy_v1/mod.rs | 30 ++-- .../network/statement-distribution/src/lib.rs | 8 +- .../statement-distribution/src/v2/mod.rs | 90 ++++++----- .../src/v2/tests/grid.rs | 2 +- 16 files changed, 264 insertions(+), 285 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index edba3e1af2da..08a5e57a9fb1 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -32,7 +32,7 @@ use polkadot_node_network_protocol::{ self as net_protocol, filter_by_peer_version, grid_topology::{RandomRouting, RequiredRouting, SessionGridTopologies, SessionGridTopology}, peer_set::MAX_NOTIFICATION_SIZE, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, PeerId, + v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, PeerId, UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_primitives::approval::{ @@ -966,16 +966,14 @@ impl State { msg: Versioned< protocol_v1::ApprovalDistributionMessage, protocol_v2::ApprovalDistributionMessage, - protocol_vstaging::ApprovalDistributionMessage, + protocol_v3::ApprovalDistributionMessage, >, rng: &mut R, ) where R: CryptoRng + Rng, { match msg { - Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Assignments( - assignments, - )) => { + Versioned::V3(protocol_v3::ApprovalDistributionMessage::Assignments(assignments)) => { gum::trace!( target: LOG_TARGET, peer_id = %peer_id, @@ -1015,9 +1013,7 @@ impl State { ) .await; }, - Versioned::VStaging(protocol_vstaging::ApprovalDistributionMessage::Approvals( - approvals, - )) => { + Versioned::V3(protocol_v3::ApprovalDistributionMessage::Approvals(approvals)) => { self.process_incoming_approvals(ctx, metrics, peer_id, approvals).await; }, Versioned::V1(protocol_v1::ApprovalDistributionMessage::Approvals(approvals)) | @@ -2458,12 +2454,12 @@ async fn send_assignments_batched_inner( peers: Vec, peer_version: ValidationVersion, ) { - if peer_version == ValidationVersion::VStaging { + if peer_version == ValidationVersion::V3 { sender .send_message(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments( + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments( batch.into_iter().collect(), ), )), @@ -2514,7 +2510,7 @@ pub(crate) async fn send_assignments_batched( ) { let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); let v2_peers = filter_by_peer_version(peers, ValidationVersion::V2.into()); - let vstaging_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + let v3_peers = filter_by_peer_version(peers, ValidationVersion::V3.into()); // V1 and V2 validation protocol do not have any changes with regard to // ApprovalDistributionMessage so they can be treated the same. @@ -2552,18 +2548,13 @@ pub(crate) async fn send_assignments_batched( } } - if !vstaging_peers.is_empty() { - let mut vstaging = v2_assignments.into_iter().peekable(); + if !v3_peers.is_empty() { + let mut v3 = v2_assignments.into_iter().peekable(); - while vstaging.peek().is_some() { - let batch = vstaging.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect::>(); - send_assignments_batched_inner( - sender, - batch, - vstaging_peers.clone(), - ValidationVersion::VStaging, - ) - .await; + while v3.peek().is_some() { + let batch = v3.by_ref().take(MAX_ASSIGNMENT_BATCH_SIZE).collect::>(); + send_assignments_batched_inner(sender, batch, v3_peers.clone(), ValidationVersion::V3) + .await; } } } @@ -2576,7 +2567,7 @@ pub(crate) async fn send_approvals_batched( ) { let v1_peers = filter_by_peer_version(peers, ValidationVersion::V1.into()); let v2_peers = filter_by_peer_version(peers, ValidationVersion::V2.into()); - let vstaging_peers = filter_by_peer_version(peers, ValidationVersion::VStaging.into()); + let v3_peers = filter_by_peer_version(peers, ValidationVersion::V3.into()); if !v1_peers.is_empty() || !v2_peers.is_empty() { let mut batches = approvals @@ -2613,7 +2604,7 @@ pub(crate) async fn send_approvals_batched( } } - if !vstaging_peers.is_empty() { + if !v3_peers.is_empty() { let mut batches = approvals.into_iter().peekable(); while batches.peek().is_some() { @@ -2621,12 +2612,10 @@ pub(crate) async fn send_approvals_batched( sender .send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_peers.clone(), - Versioned::VStaging( - protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(batch), - ), - ), + v3_peers.clone(), + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(batch), + )), )) .await; } diff --git a/polkadot/node/network/approval-distribution/src/tests.rs b/polkadot/node/network/approval-distribution/src/tests.rs index b221d6aeef76..ad5d0bb0a9c5 100644 --- a/polkadot/node/network/approval-distribution/src/tests.rs +++ b/polkadot/node/network/approval-distribution/src/tests.rs @@ -276,16 +276,16 @@ async fn send_message_from_peer_v2( .await; } -async fn send_message_from_peer_vstaging( +async fn send_message_from_peer_v3( virtual_overseer: &mut VirtualOverseer, peer_id: &PeerId, - msg: protocol_vstaging::ApprovalDistributionMessage, + msg: protocol_v3::ApprovalDistributionMessage, ) { overseer_send( virtual_overseer, ApprovalDistributionMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage( *peer_id, - Versioned::VStaging(msg), + Versioned::V3(msg), )), ) .await; @@ -450,7 +450,7 @@ fn try_import_the_same_assignment() { ); // setup new peer with V2 - setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::V3).await; // send the same assignment from peer_d let msg = protocol_v1::ApprovalDistributionMessage::Assignments(assignments); @@ -479,9 +479,9 @@ fn try_import_the_same_assignment_v2() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers - setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; // Set up a gossip topology, where a, b, c and d are topology neighboors to the node under // testing. @@ -512,8 +512,8 @@ fn try_import_the_same_assignment_v2() { let cert = fake_assignment_cert_v2(hash, validator_index, core_bitfield.clone()); let assignments = vec![(cert.clone(), cores.clone().try_into().unwrap())]; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments.clone()); - send_message_from_peer_vstaging(overseer, &peer_a, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments.clone()); + send_message_from_peer_v3(overseer, &peer_a, msg).await; expect_reputation_change(overseer, &peer_a, COST_UNEXPECTED_MESSAGE).await; @@ -537,8 +537,8 @@ fn try_import_the_same_assignment_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert_eq!(peers.len(), 2); @@ -547,11 +547,11 @@ fn try_import_the_same_assignment_v2() { ); // setup new peer - setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_d, view![], ValidationVersion::V3).await; // send the same assignment from peer_d - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(assignments); + send_message_from_peer_v3(overseer, &peer_d, msg).await; expect_reputation_change(overseer, &peer_d, COST_UNEXPECTED_MESSAGE).await; expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE).await; @@ -807,7 +807,7 @@ fn import_approval_happy_path_v1_v2_peers() { let overseer = &mut virtual_overseer; // setup peers with V1 and V2 protocol versions setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V1).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V1).await; // new block `hash_a` with 1 candidates @@ -857,8 +857,8 @@ fn import_approval_happy_path_v1_v2_peers() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert_eq!(peers.len(), 1); @@ -873,9 +873,9 @@ fn import_approval_happy_path_v1_v2_peers() { validator: validator_index, signature: dummy_signature(), }; - let msg: protocol_vstaging::ApprovalDistributionMessage = - protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_b, msg).await; + let msg: protocol_v3::ApprovalDistributionMessage = + protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -920,9 +920,9 @@ fn import_approval_happy_path_v2() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers with V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_a, view![], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -959,8 +959,8 @@ fn import_approval_happy_path_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert_eq!(peers.len(), 2); @@ -975,8 +975,8 @@ fn import_approval_happy_path_v2() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_b, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -995,8 +995,8 @@ fn import_approval_happy_path_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) )) )) => { assert_eq!(peers.len(), 1); @@ -1022,10 +1022,10 @@ fn multiple_assignments_covered_with_one_approval_vote() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; // setup peers with V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_c, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::V3).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -1056,11 +1056,11 @@ fn multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, cert: cert.cert, }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( assignment, (0 as CandidateIndex).into(), )]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1077,8 +1077,8 @@ fn multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert!(peers.len() >= 2); @@ -1097,12 +1097,12 @@ fn multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, cert: cert.cert, }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( assignment, (1 as CandidateIndex).into(), )]); - send_message_from_peer_vstaging(overseer, &peer_c, msg).await; + send_message_from_peer_v3(overseer, &peer_c, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1119,8 +1119,8 @@ fn multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert!(peers.len() >= 2); @@ -1137,8 +1137,8 @@ fn multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1157,8 +1157,8 @@ fn multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) )) )) => { assert!(peers.len() >= 2); @@ -1207,7 +1207,7 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { let _ = test_harness(state_without_reputation_delay(), |mut virtual_overseer| async move { let overseer = &mut virtual_overseer; - setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_d, view![hash], ValidationVersion::V3).await; // new block `hash_a` with 1 candidates let meta = BlockApprovalMeta { @@ -1238,11 +1238,11 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, cert: cert.cert, }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( assignment, (0 as CandidateIndex).into(), )]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1264,12 +1264,12 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, cert: cert.cert, }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Assignments(vec![( + let msg = protocol_v3::ApprovalDistributionMessage::Assignments(vec![( assignment, (1 as CandidateIndex).into(), )]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1289,8 +1289,8 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_d, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_d, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1306,16 +1306,16 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { expect_reputation_change(overseer, &peer_d, BENEFIT_VALID_MESSAGE_FIRST).await; // setup peers with V2 protocol versions - setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::VStaging).await; - setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, &peer_a, view![hash], ValidationVersion::V3).await; + setup_peer_with_view(overseer, &peer_b, view![hash], ValidationVersion::V3).await; let mut expected_peers_assignments = vec![peer_a, peer_b]; let mut expected_peers_approvals = vec![peer_a, peer_b]; assert_matches!( overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert!(peers.len() == 1); @@ -1329,8 +1329,8 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) )) )) => { assert!(peers.len() == 1); @@ -1344,8 +1344,8 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(assignments) )) )) => { assert!(peers.len() == 1); @@ -1359,8 +1359,8 @@ fn unify_with_peer_multiple_assignments_covered_with_one_approval_vote() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(approvals) )) )) => { assert!(peers.len() == 1); @@ -1410,8 +1410,8 @@ fn import_approval_bad() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_b, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; expect_reputation_change(overseer, &peer_b, COST_UNEXPECTED_MESSAGE).await; @@ -1436,8 +1436,8 @@ fn import_approval_bad() { expect_reputation_change(overseer, &peer_b, BENEFIT_VALID_MESSAGE_FIRST).await; // and try again - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, &peer_b, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, &peer_b, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1762,8 +1762,8 @@ fn import_remotely_then_locally() { validator: validator_index, signature: dummy_signature(), }; - let msg = protocol_vstaging::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); - send_message_from_peer_vstaging(overseer, peer, msg).await; + let msg = protocol_v3::ApprovalDistributionMessage::Approvals(vec![approval.clone()]); + send_message_from_peer_v3(overseer, peer, msg).await; assert_matches!( overseer_recv(overseer).await, @@ -1945,7 +1945,7 @@ fn sends_assignments_even_when_state_is_approved_v2() { } // connect the peer. - setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::VStaging).await; + setup_peer_with_view(overseer, peer, view![hash], ValidationVersion::V3).await; let assignments = vec![(cert.clone(), candidate_bitfield.clone())]; @@ -1953,8 +1953,8 @@ fn sends_assignments_even_when_state_is_approved_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Assignments(sent_assignments) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Assignments(sent_assignments) )) )) => { assert_eq!(peers, vec![*peer]); @@ -1966,8 +1966,8 @@ fn sends_assignments_even_when_state_is_approved_v2() { overseer_recv(overseer).await, AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::SendValidationMessage( peers, - Versioned::VStaging(protocol_vstaging::ValidationProtocol::ApprovalDistribution( - protocol_vstaging::ApprovalDistributionMessage::Approvals(sent_approvals) + Versioned::V3(protocol_v3::ValidationProtocol::ApprovalDistribution( + protocol_v3::ApprovalDistributionMessage::Approvals(sent_approvals) )) )) => { // Construct a hashmaps of approvals for comparison. Approval distribution reorders messages because they are kept in a diff --git a/polkadot/node/network/bitfield-distribution/src/lib.rs b/polkadot/node/network/bitfield-distribution/src/lib.rs index 9cc79aee8490..76baf499cad7 100644 --- a/polkadot/node/network/bitfield-distribution/src/lib.rs +++ b/polkadot/node/network/bitfield-distribution/src/lib.rs @@ -32,7 +32,7 @@ use polkadot_node_network_protocol::{ GridNeighbors, RandomRouting, RequiredRouting, SessionBoundGridTopologyStorage, }, peer_set::{ProtocolVersion, ValidationVersion}, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, OurView, PeerId, + v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, OurView, PeerId, UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_subsystem::{ @@ -102,8 +102,8 @@ impl BitfieldGossipMessage { self.relay_parent, self.signed_availability.into(), )), - Some(ValidationVersion::VStaging) => - Versioned::VStaging(protocol_vstaging::BitfieldDistributionMessage::Bitfield( + Some(ValidationVersion::V3) => + Versioned::V3(protocol_v3::BitfieldDistributionMessage::Bitfield( self.relay_parent, self.signed_availability.into(), )), @@ -503,8 +503,8 @@ async fn relay_message( let v2_interested_peers = filter_by_peer_version(&interested_peers, ValidationVersion::V2.into()); - let vstaging_interested_peers = - filter_by_peer_version(&interested_peers, ValidationVersion::VStaging.into()); + let v3_interested_peers = + filter_by_peer_version(&interested_peers, ValidationVersion::V3.into()); if !v1_interested_peers.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( @@ -522,10 +522,10 @@ async fn relay_message( .await } - if !vstaging_interested_peers.is_empty() { + if !v3_interested_peers.is_empty() { ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_interested_peers, - message.into_validation_protocol(ValidationVersion::VStaging.into()), + v3_interested_peers, + message.into_validation_protocol(ValidationVersion::V3.into()), )) .await } @@ -551,7 +551,7 @@ async fn process_incoming_peer_message( relay_parent, bitfield, )) | - Versioned::VStaging(protocol_vstaging::BitfieldDistributionMessage::Bitfield( + Versioned::V3(protocol_v3::BitfieldDistributionMessage::Bitfield( relay_parent, bitfield, )) => (relay_parent, bitfield), diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index a9339a5c443c..2fcf5cec489d 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -33,7 +33,7 @@ use sc_network::{ use polkadot_node_network_protocol::{ peer_set::{CollationVersion, PeerSet, ProtocolVersion, ValidationVersion}, request_response::{OutgoingRequest, Recipient, ReqProtocolNames, Requests}, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, PeerId, + v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, PeerId, }; use polkadot_primitives::{AuthorityDiscoveryId, Block, Hash}; @@ -62,20 +62,20 @@ pub(crate) fn send_validation_message_v1( ); } -// Helper function to send a validation vstaging message to a list of peers. +// Helper function to send a validation v3 message to a list of peers. // Messages are always sent via the main protocol, even legacy protocol messages. -pub(crate) fn send_validation_message_vstaging( +pub(crate) fn send_validation_message_v3( peers: Vec, - message: WireMessage, + message: WireMessage, metrics: &Metrics, notification_sinks: &Arc>>>, ) { - gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation vstaging message to peers",); + gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v3 message to peers",); send_message( peers, PeerSet::Validation, - ValidationVersion::VStaging.into(), + ValidationVersion::V3.into(), message, metrics, notification_sinks, diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 40cd167a968b..49d81aea76af 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -37,8 +37,8 @@ use polkadot_node_network_protocol::{ CollationVersion, PeerSet, PeerSetProtocolNames, PerPeerSet, ProtocolVersion, ValidationVersion, }, - v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, ObservedRole, OurView, - PeerId, UnifiedReputationChange as Rep, View, + v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, ObservedRole, OurView, PeerId, + UnifiedReputationChange as Rep, View, }; use polkadot_node_subsystem::{ @@ -70,7 +70,7 @@ use super::validator_discovery; /// Defines the `Network` trait with an implementation for an `Arc`. use crate::network::{ send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, - send_validation_message_v2, send_validation_message_vstaging, Network, + send_validation_message_v2, send_validation_message_v3, Network, }; use crate::{network::get_peer_id_by_authority_id, WireMessage}; @@ -294,9 +294,9 @@ async fn handle_validation_message( metrics, notification_sinks, ), - ValidationVersion::VStaging => send_validation_message_vstaging( + ValidationVersion::V3 => send_validation_message_v3( vec![peer], - WireMessage::::ViewUpdate(local_view), + WireMessage::::ViewUpdate(local_view), metrics, notification_sinks, ), @@ -360,48 +360,47 @@ async fn handle_validation_message( ?peer, ); - let (events, reports) = - if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V1.into()) { - handle_peer_messages::( - peer, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - vec![notification.into()], - metrics, - ) - } else if expected_versions[PeerSet::Validation] == - Some(ValidationVersion::V2.into()) - { - handle_peer_messages::( - peer, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - vec![notification.into()], - metrics, - ) - } else if expected_versions[PeerSet::Validation] == - Some(ValidationVersion::VStaging.into()) - { - handle_peer_messages::( - peer, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - vec![notification.into()], - metrics, - ) - } else { - gum::warn!( - target: LOG_TARGET, - version = ?expected_versions[PeerSet::Validation], - "Major logic bug. Peer somehow has unsupported validation protocol version." - ); + let (events, reports) = if expected_versions[PeerSet::Validation] == + Some(ValidationVersion::V1.into()) + { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V2.into()) { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V3.into()) { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else { + gum::warn!( + target: LOG_TARGET, + version = ?expected_versions[PeerSet::Validation], + "Major logic bug. Peer somehow has unsupported validation protocol version." + ); - never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); + never!( + "Only versions 1 and 2 are supported; peer set connection checked above; qed" + ); - // If a peer somehow triggers this, we'll disconnect them - // eventually. - (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) - }; + // If a peer somehow triggers this, we'll disconnect them + // eventually. + (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) + }; for report in reports { network_service.report_peer(peer, report.into()); @@ -980,8 +979,8 @@ fn update_our_view( filter_by_peer_version(&validation_peers, ValidationVersion::V2.into()); let v2_collation_peers = filter_by_peer_version(&collation_peers, CollationVersion::V2.into()); - let vstaging_validation_peers = - filter_by_peer_version(&validation_peers, ValidationVersion::VStaging.into()); + let v3_validation_peers = + filter_by_peer_version(&validation_peers, ValidationVersion::V3.into()); send_validation_message_v1( v1_validation_peers, @@ -1011,8 +1010,8 @@ fn update_our_view( notification_sinks, ); - send_validation_message_vstaging( - vstaging_validation_peers, + send_validation_message_v3( + v3_validation_peers, WireMessage::ViewUpdate(new_view.clone()), metrics, notification_sinks, diff --git a/polkadot/node/network/bridge/src/rx/tests.rs b/polkadot/node/network/bridge/src/rx/tests.rs index e0b86feb6442..ad503ee21f53 100644 --- a/polkadot/node/network/bridge/src/rx/tests.rs +++ b/polkadot/node/network/bridge/src/rx/tests.rs @@ -224,7 +224,7 @@ impl TestNetworkHandle { PeerSet::Validation => Some(ProtocolName::from("/polkadot/validation/1")), PeerSet::Collation => Some(ProtocolName::from("/polkadot/collation/1")), }, - ValidationVersion::VStaging => match peer_set { + ValidationVersion::V3 => match peer_set { PeerSet::Validation => Some(ProtocolName::from("/polkadot/validation/3")), PeerSet::Collation => unreachable!(), }, @@ -1433,8 +1433,8 @@ fn network_protocol_versioning_view_update() { ValidationVersion::V2 => WireMessage::::ViewUpdate(view.clone()) .encode(), - ValidationVersion::VStaging => - WireMessage::::ViewUpdate(view.clone()) + ValidationVersion::V3 => + WireMessage::::ViewUpdate(view.clone()) .encode(), }; assert_network_actions_contains( diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index bdcd1574e335..22802608e1d5 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -41,7 +41,7 @@ use crate::validator_discovery; /// Defines the `Network` trait with an implementation for an `Arc`. use crate::network::{ send_collation_message_v1, send_collation_message_v2, send_validation_message_v1, - send_validation_message_v2, send_validation_message_vstaging, Network, + send_validation_message_v2, send_validation_message_v3, Network, }; use crate::metrics::Metrics; @@ -205,7 +205,7 @@ where &metrics, notification_sinks, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::V3(msg) => send_validation_message_v3( peers, WireMessage::ProtocolMessage(msg), &metrics, @@ -235,7 +235,7 @@ where &metrics, notification_sinks, ), - Versioned::VStaging(msg) => send_validation_message_vstaging( + Versioned::V3(msg) => send_validation_message_v3( peers, WireMessage::ProtocolMessage(msg), &metrics, @@ -264,7 +264,7 @@ where &metrics, notification_sinks, ), - Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( + Versioned::V2(msg) | Versioned::V3(msg) => send_collation_message_v2( peers, WireMessage::ProtocolMessage(msg), &metrics, @@ -287,7 +287,7 @@ where &metrics, notification_sinks, ), - Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2( + Versioned::V2(msg) | Versioned::V3(msg) => send_collation_message_v2( peers, WireMessage::ProtocolMessage(msg), &metrics, diff --git a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs index b3a396e1be34..8fb0bb215444 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs @@ -882,7 +882,7 @@ async fn handle_incoming_peer_message( match msg { Versioned::V1(V1::Declare(..)) | Versioned::V2(V2::Declare(..)) | - Versioned::VStaging(V2::Declare(..)) => { + Versioned::V3(V2::Declare(..)) => { gum::trace!( target: LOG_TARGET, ?origin, @@ -895,7 +895,7 @@ async fn handle_incoming_peer_message( }, Versioned::V1(V1::AdvertiseCollation(_)) | Versioned::V2(V2::AdvertiseCollation { .. }) | - Versioned::VStaging(V2::AdvertiseCollation { .. }) => { + Versioned::V3(V2::AdvertiseCollation { .. }) => { gum::trace!( target: LOG_TARGET, ?origin, @@ -911,7 +911,7 @@ async fn handle_incoming_peer_message( }, Versioned::V1(V1::CollationSeconded(relay_parent, statement)) | Versioned::V2(V2::CollationSeconded(relay_parent, statement)) | - Versioned::VStaging(V2::CollationSeconded(relay_parent, statement)) => { + Versioned::V3(V2::CollationSeconded(relay_parent, statement)) => { if !matches!(statement.unchecked_payload(), Statement::Seconded(_)) { gum::warn!( target: LOG_TARGET, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 20b3b9ea1d26..48ad3c711a6d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -777,7 +777,7 @@ async fn process_incoming_peer_message( match msg { Versioned::V1(V1::Declare(collator_id, para_id, signature)) | Versioned::V2(V2::Declare(collator_id, para_id, signature)) | - Versioned::VStaging(V2::Declare(collator_id, para_id, signature)) => { + Versioned::V3(V2::Declare(collator_id, para_id, signature)) => { if collator_peer_id(&state.peer_data, &collator_id).is_some() { modify_reputation( &mut state.reputation, @@ -894,7 +894,7 @@ async fn process_incoming_peer_message( candidate_hash, parent_head_data_hash, }) | - Versioned::VStaging(V2::AdvertiseCollation { + Versioned::V3(V2::AdvertiseCollation { relay_parent, candidate_hash, parent_head_data_hash, @@ -923,7 +923,7 @@ async fn process_incoming_peer_message( }, Versioned::V1(V1::CollationSeconded(..)) | Versioned::V2(V2::CollationSeconded(..)) | - Versioned::VStaging(V2::CollationSeconded(..)) => { + Versioned::V3(V2::CollationSeconded(..)) => { gum::warn!( target: LOG_TARGET, peer_id = ?origin, diff --git a/polkadot/node/network/gossip-support/src/lib.rs b/polkadot/node/network/gossip-support/src/lib.rs index 0d1b04f2ba23..22417795d5ea 100644 --- a/polkadot/node/network/gossip-support/src/lib.rs +++ b/polkadot/node/network/gossip-support/src/lib.rs @@ -477,7 +477,7 @@ where match message { Versioned::V1(m) => match m {}, Versioned::V2(m) => match m {}, - Versioned::VStaging(m) => match m {}, + Versioned::V3(m) => match m {}, } }, } diff --git a/polkadot/node/network/protocol/src/lib.rs b/polkadot/node/network/protocol/src/lib.rs index b0e30701208b..ae72230ee43d 100644 --- a/polkadot/node/network/protocol/src/lib.rs +++ b/polkadot/node/network/protocol/src/lib.rs @@ -253,29 +253,29 @@ impl View { /// A protocol-versioned type. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Versioned { +pub enum Versioned { /// V1 type. V1(V1), /// V2 type. V2(V2), - /// VStaging type - VStaging(VStaging), + /// V3 type + V3(V3), } -impl Versioned<&'_ V1, &'_ V2, &'_ VStaging> { +impl Versioned<&'_ V1, &'_ V2, &'_ V3> { /// Convert to a fully-owned version of the message. - pub fn clone_inner(&self) -> Versioned { + pub fn clone_inner(&self) -> Versioned { match *self { Versioned::V1(inner) => Versioned::V1(inner.clone()), Versioned::V2(inner) => Versioned::V2(inner.clone()), - Versioned::VStaging(inner) => Versioned::VStaging(inner.clone()), + Versioned::V3(inner) => Versioned::V3(inner.clone()), } } } /// All supported versions of the validation protocol message. pub type VersionedValidationProtocol = - Versioned; + Versioned; impl From for VersionedValidationProtocol { fn from(v1: v1::ValidationProtocol) -> Self { @@ -289,9 +289,9 @@ impl From for VersionedValidationProtocol { } } -impl From for VersionedValidationProtocol { - fn from(vstaging: vstaging::ValidationProtocol) -> Self { - VersionedValidationProtocol::VStaging(vstaging) +impl From for VersionedValidationProtocol { + fn from(v3: v3::ValidationProtocol) -> Self { + VersionedValidationProtocol::V3(v3) } } @@ -317,7 +317,7 @@ macro_rules! impl_versioned_full_protocol_from { match versioned_from { Versioned::V1(x) => Versioned::V1(x.into()), Versioned::V2(x) => Versioned::V2(x.into()), - Versioned::VStaging(x) => Versioned::VStaging(x.into()), + Versioned::V3(x) => Versioned::V3(x.into()), } } } @@ -331,7 +331,7 @@ macro_rules! impl_versioned_try_from { $out:ty, $v1_pat:pat => $v1_out:expr, $v2_pat:pat => $v2_out:expr, - $vstaging_pat:pat => $vstaging_out:expr + $v3_pat:pat => $v3_out:expr ) => { impl TryFrom<$from> for $out { type Error = crate::WrongVariant; @@ -341,7 +341,7 @@ macro_rules! impl_versioned_try_from { match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out)), Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out)), - Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out)), + Versioned::V3($v3_pat) => Ok(Versioned::V3($v3_out)), _ => Err(crate::WrongVariant), } } @@ -355,8 +355,7 @@ macro_rules! impl_versioned_try_from { match x { Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out.clone())), Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out.clone())), - Versioned::VStaging($vstaging_pat) => - Ok(Versioned::VStaging($vstaging_out.clone())), + Versioned::V3($v3_pat) => Ok(Versioned::V3($v3_out.clone())), _ => Err(crate::WrongVariant), } } @@ -368,7 +367,7 @@ macro_rules! impl_versioned_try_from { pub type BitfieldDistributionMessage = Versioned< v1::BitfieldDistributionMessage, v2::BitfieldDistributionMessage, - vstaging::BitfieldDistributionMessage, + v3::BitfieldDistributionMessage, >; impl_versioned_full_protocol_from!( BitfieldDistributionMessage, @@ -380,14 +379,14 @@ impl_versioned_try_from!( BitfieldDistributionMessage, v1::ValidationProtocol::BitfieldDistribution(x) => x, v2::ValidationProtocol::BitfieldDistribution(x) => x, - vstaging::ValidationProtocol::BitfieldDistribution(x) => x + v3::ValidationProtocol::BitfieldDistribution(x) => x ); /// Version-annotated messages used by the statement distribution subsystem. pub type StatementDistributionMessage = Versioned< v1::StatementDistributionMessage, v2::StatementDistributionMessage, - vstaging::StatementDistributionMessage, + v3::StatementDistributionMessage, >; impl_versioned_full_protocol_from!( StatementDistributionMessage, @@ -399,14 +398,14 @@ impl_versioned_try_from!( StatementDistributionMessage, v1::ValidationProtocol::StatementDistribution(x) => x, v2::ValidationProtocol::StatementDistribution(x) => x, - vstaging::ValidationProtocol::StatementDistribution(x) => x + v3::ValidationProtocol::StatementDistribution(x) => x ); /// Version-annotated messages used by the approval distribution subsystem. pub type ApprovalDistributionMessage = Versioned< v1::ApprovalDistributionMessage, v2::ApprovalDistributionMessage, - vstaging::ApprovalDistributionMessage, + v3::ApprovalDistributionMessage, >; impl_versioned_full_protocol_from!( ApprovalDistributionMessage, @@ -418,7 +417,7 @@ impl_versioned_try_from!( ApprovalDistributionMessage, v1::ValidationProtocol::ApprovalDistribution(x) => x, v2::ValidationProtocol::ApprovalDistribution(x) => x, - vstaging::ValidationProtocol::ApprovalDistribution(x) => x + v3::ValidationProtocol::ApprovalDistribution(x) => x ); @@ -426,7 +425,7 @@ impl_versioned_try_from!( pub type GossipSupportNetworkMessage = Versioned< v1::GossipSupportNetworkMessage, v2::GossipSupportNetworkMessage, - vstaging::GossipSupportNetworkMessage, + v3::GossipSupportNetworkMessage, >; // This is a void enum placeholder, so never gets sent over the wire. @@ -871,18 +870,17 @@ pub mod v2 { } } -/// vstaging network protocol types, intended to become v3. -/// Initial purpose is for chaning ApprovalDistributionMessage to -/// include more than one assignment in the message. -pub mod vstaging { +/// v3 network protocol types. +/// Purpose is for chaning ApprovalDistributionMessage to +/// include more than one assignment and approval in a message. +pub mod v3 { use parity_scale_codec::{Decode, Encode}; use polkadot_node_primitives::approval::v2::{ CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2, }; - /// This parts of the protocol did not change from v2, so just alias them in vstaging, - /// no reason why they can't be change untill vstaging becomes v3 and is released. + /// This parts of the protocol did not change from v2, so just alias them in v3. pub use super::v2::{ declare_signature_payload, BackedCandidateAcknowledgement, BackedCandidateManifest, BitfieldDistributionMessage, GossipSupportNetworkMessage, StatementDistributionMessage, diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index 7e257d508b5b..fb2e2d77f6ff 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -135,7 +135,7 @@ impl PeerSet { #[cfg(feature = "network-protocol-staging")] match self { - PeerSet::Validation => ValidationVersion::VStaging.into(), + PeerSet::Validation => ValidationVersion::V3.into(), PeerSet::Collation => CollationVersion::V2.into(), } } @@ -163,7 +163,7 @@ impl PeerSet { Some("validation/1") } else if version == ValidationVersion::V2.into() { Some("validation/2") - } else if version == ValidationVersion::VStaging.into() { + } else if version == ValidationVersion::V3.into() { Some("validation/3") } else { None @@ -236,9 +236,10 @@ pub enum ValidationVersion { V1 = 1, /// The second version. V2 = 2, - /// The staging version to gather changes - /// that before the release become v3. - VStaging = 3, + /// The third version where changes to ApprovalDistributionMessage had been made. + /// The changes are translatable to V2 format untill assignments v2 and approvals + /// coalescing is enabled through a runtime upgrade. + V3 = 3, } /// Supported collation protocol versions. Only versions defined here must be used in the codebase. diff --git a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs index d9866af1ee23..93f97fe1dd6e 100644 --- a/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs +++ b/polkadot/node/network/statement-distribution/src/legacy_v1/mod.rs @@ -22,8 +22,8 @@ use polkadot_node_network_protocol::{ grid_topology::{GridNeighbors, RequiredRouting, SessionBoundGridTopologyStorage}, peer_set::{IsAuthority, PeerSet, ValidationVersion}, v1::{self as protocol_v1, StatementMetadata}, - v2 as protocol_v2, vstaging as protocol_vstaging, IfDisconnected, PeerId, - UnifiedReputationChange as Rep, Versioned, View, + v2 as protocol_v2, v3 as protocol_v3, IfDisconnected, PeerId, UnifiedReputationChange as Rep, + Versioned, View, }; use polkadot_node_primitives::{ SignedFullStatement, Statement, StatementWithPVD, UncheckedSignedFullStatement, @@ -1075,7 +1075,7 @@ async fn circulate_statement<'a, Context>( }) .partition::, _>(|(_, _, version)| match version { ValidationVersion::V1 => true, - ValidationVersion::V2 | ValidationVersion::VStaging => false, + ValidationVersion::V2 | ValidationVersion::V3 => false, }); // partition is handy here but not if we add more protocol versions let payload = v1_statement_message(relay_parent, stored.statement.clone(), metrics); @@ -1108,8 +1108,7 @@ async fn circulate_statement<'a, Context>( .collect(); let v2_peers_to_send = filter_by_peer_version(&peers_to_send, ValidationVersion::V2.into()); - let vstaging_to_send = - filter_by_peer_version(&peers_to_send, ValidationVersion::VStaging.into()); + let v3_to_send = filter_by_peer_version(&peers_to_send, ValidationVersion::V3.into()); if !v2_peers_to_send.is_empty() { gum::trace!( @@ -1126,17 +1125,17 @@ async fn circulate_statement<'a, Context>( .await; } - if !vstaging_to_send.is_empty() { + if !v3_to_send.is_empty() { gum::trace!( target: LOG_TARGET, - ?vstaging_to_send, + ?v3_to_send, ?relay_parent, statement = ?stored.statement, - "Sending statement to vstaging peers", + "Sending statement to v3 peers", ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - vstaging_to_send, - compatible_v1_message(ValidationVersion::VStaging, payload.clone()).into(), + v3_to_send, + compatible_v1_message(ValidationVersion::V3, payload.clone()).into(), )) .await; } @@ -1472,10 +1471,8 @@ async fn handle_incoming_message<'a, Context>( let message = match message { Versioned::V1(m) => m, Versioned::V2(protocol_v2::StatementDistributionMessage::V1Compatibility(m)) | - Versioned::VStaging(protocol_vstaging::StatementDistributionMessage::V1Compatibility( - m, - )) => m, - Versioned::V2(_) | Versioned::VStaging(_) => { + Versioned::V3(protocol_v3::StatementDistributionMessage::V1Compatibility(m)) => m, + Versioned::V2(_) | Versioned::V3(_) => { // The higher-level subsystem code is supposed to filter out // all non v1 messages. gum::debug!( @@ -2201,8 +2198,7 @@ fn compatible_v1_message( ValidationVersion::V1 => Versioned::V1(message), ValidationVersion::V2 => Versioned::V2(protocol_v2::StatementDistributionMessage::V1Compatibility(message)), - ValidationVersion::VStaging => Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::V1Compatibility(message), - ), + ValidationVersion::V3 => + Versioned::V3(protocol_v3::StatementDistributionMessage::V1Compatibility(message)), } } diff --git a/polkadot/node/network/statement-distribution/src/lib.rs b/polkadot/node/network/statement-distribution/src/lib.rs index ef1fc7cd78b5..a1ba1137b5ac 100644 --- a/polkadot/node/network/statement-distribution/src/lib.rs +++ b/polkadot/node/network/statement-distribution/src/lib.rs @@ -27,7 +27,7 @@ use std::time::Duration; use polkadot_node_network_protocol::{ request_response::{v1 as request_v1, v2::AttestedCandidateRequest, IncomingRequestReceiver}, - v2 as protocol_v2, vstaging as protocol_vstaging, Versioned, + v2 as protocol_v2, v3 as protocol_v3, Versioned, }; use polkadot_node_primitives::StatementWithPVD; use polkadot_node_subsystem::{ @@ -400,11 +400,11 @@ impl StatementDistributionSubsystem { Versioned::V2( protocol_v2::StatementDistributionMessage::V1Compatibility(_), ) | - Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::V1Compatibility(_), + Versioned::V3( + protocol_v3::StatementDistributionMessage::V1Compatibility(_), ) => VersionTarget::Legacy, Versioned::V1(_) => VersionTarget::Legacy, - Versioned::V2(_) | Versioned::VStaging(_) => VersionTarget::Current, + Versioned::V2(_) | Versioned::V3(_) => VersionTarget::Current, }, _ => VersionTarget::Both, }; diff --git a/polkadot/node/network/statement-distribution/src/v2/mod.rs b/polkadot/node/network/statement-distribution/src/v2/mod.rs index 406f11305909..2f06d3685b81 100644 --- a/polkadot/node/network/statement-distribution/src/v2/mod.rs +++ b/polkadot/node/network/statement-distribution/src/v2/mod.rs @@ -29,8 +29,7 @@ use polkadot_node_network_protocol::{ MAX_PARALLEL_ATTESTED_CANDIDATE_REQUESTS, }, v2::{self as protocol_v2, StatementFilter}, - vstaging as protocol_vstaging, IfDisconnected, PeerId, UnifiedReputationChange as Rep, - Versioned, View, + v3 as protocol_v3, IfDisconnected, PeerId, UnifiedReputationChange as Rep, Versioned, View, }; use polkadot_node_primitives::{ SignedFullStatementWithPVD, StatementWithPVD as FullStatementWithPVD, @@ -366,7 +365,7 @@ pub(crate) async fn handle_network_update( gum::trace!(target: LOG_TARGET, ?peer_id, ?role, ?protocol_version, "Peer connected"); let versioned_protocol = if protocol_version != ValidationVersion::V2.into() && - protocol_version != ValidationVersion::VStaging.into() + protocol_version != ValidationVersion::V3.into() { return } else { @@ -432,28 +431,28 @@ pub(crate) async fn handle_network_update( net_protocol::StatementDistributionMessage::V2( protocol_v2::StatementDistributionMessage::V1Compatibility(_), ) | - net_protocol::StatementDistributionMessage::VStaging( - protocol_vstaging::StatementDistributionMessage::V1Compatibility(_), + net_protocol::StatementDistributionMessage::V3( + protocol_v3::StatementDistributionMessage::V1Compatibility(_), ) => return, net_protocol::StatementDistributionMessage::V2( protocol_v2::StatementDistributionMessage::Statement(relay_parent, statement), ) | - net_protocol::StatementDistributionMessage::VStaging( - protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, statement), + net_protocol::StatementDistributionMessage::V3( + protocol_v3::StatementDistributionMessage::Statement(relay_parent, statement), ) => handle_incoming_statement(ctx, state, peer_id, relay_parent, statement, reputation) .await, net_protocol::StatementDistributionMessage::V2( protocol_v2::StatementDistributionMessage::BackedCandidateManifest(inner), ) | - net_protocol::StatementDistributionMessage::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(inner), + net_protocol::StatementDistributionMessage::V3( + protocol_v3::StatementDistributionMessage::BackedCandidateManifest(inner), ) => handle_incoming_manifest(ctx, state, peer_id, inner, reputation).await, net_protocol::StatementDistributionMessage::V2( protocol_v2::StatementDistributionMessage::BackedCandidateKnown(inner), ) | - net_protocol::StatementDistributionMessage::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateKnown(inner), + net_protocol::StatementDistributionMessage::V3( + protocol_v3::StatementDistributionMessage::BackedCandidateKnown(inner), ) => handle_incoming_acknowledgement(ctx, state, peer_id, inner, reputation).await, }, NetworkBridgeEvent::PeerViewChange(peer_id, view) => @@ -806,13 +805,13 @@ fn pending_statement_network_message( protocol_v2::StatementDistributionMessage::Statement(relay_parent, signed) }) .map(|msg| (vec![peer.0], Versioned::V2(msg).into())), - ValidationVersion::VStaging => statement_store + ValidationVersion::V3 => statement_store .validator_statement(originator, compact) .map(|s| s.as_unchecked().clone()) .map(|signed| { - protocol_vstaging::StatementDistributionMessage::Statement(relay_parent, signed) + protocol_v3::StatementDistributionMessage::Statement(relay_parent, signed) }) - .map(|msg| (vec![peer.0], Versioned::VStaging(msg).into())), + .map(|msg| (vec![peer.0], Versioned::V3(msg).into())), ValidationVersion::V1 => { gum::error!( target: LOG_TARGET, @@ -945,10 +944,10 @@ async fn send_pending_grid_messages( ) .into(), )), - ValidationVersion::VStaging => messages.push(( + ValidationVersion::V3 => messages.push(( vec![peer_id.0], - Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest( + Versioned::V3( + protocol_v3::StatementDistributionMessage::BackedCandidateManifest( manifest, ), ) @@ -960,7 +959,7 @@ async fn send_pending_grid_messages( "Bug ValidationVersion::V1 should not be used in statement-distribution v2, legacy should have handled this" ); - } + }, }; }, grid::ManifestKind::Acknowledgement => { @@ -1308,8 +1307,8 @@ async fn circulate_statement( let statement_to_v2_peers = filter_by_peer_version(&statement_to_peers, ValidationVersion::V2.into()); - let statement_to_vstaging_peers = - filter_by_peer_version(&statement_to_peers, ValidationVersion::VStaging.into()); + let statement_to_v3_peers = + filter_by_peer_version(&statement_to_peers, ValidationVersion::V3.into()); // ship off the network messages to the network bridge. if !statement_to_v2_peers.is_empty() { @@ -1331,17 +1330,17 @@ async fn circulate_statement( .await; } - if !statement_to_vstaging_peers.is_empty() { + if !statement_to_v3_peers.is_empty() { gum::debug!( target: LOG_TARGET, ?compact_statement, n_peers = ?statement_to_peers.len(), - "Sending statement to vstaging peers", + "Sending statement to v3 peers", ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - statement_to_vstaging_peers, - Versioned::VStaging(protocol_vstaging::StatementDistributionMessage::Statement( + statement_to_v3_peers, + Versioned::V3(protocol_v3::StatementDistributionMessage::Statement( relay_parent, statement.as_unchecked().clone(), )) @@ -1887,8 +1886,7 @@ async fn provide_candidate_to_grid( } let manifest_peers_v2 = filter_by_peer_version(&manifest_peers, ValidationVersion::V2.into()); - let manifest_peers_vstaging = - filter_by_peer_version(&manifest_peers, ValidationVersion::VStaging.into()); + let manifest_peers_v3 = filter_by_peer_version(&manifest_peers, ValidationVersion::V3.into()); if !manifest_peers_v2.is_empty() { gum::debug!( target: LOG_TARGET, @@ -1908,27 +1906,27 @@ async fn provide_candidate_to_grid( .await; } - if !manifest_peers_vstaging.is_empty() { + if !manifest_peers_v3.is_empty() { gum::debug!( target: LOG_TARGET, ?candidate_hash, local_validator = ?per_session.local_validator, - n_peers = manifest_peers_vstaging.len(), - "Sending manifest to vstaging peers" + n_peers = manifest_peers_v3.len(), + "Sending manifest to v3 peers" ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - manifest_peers_vstaging, - Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(manifest), - ) + manifest_peers_v3, + Versioned::V3(protocol_v3::StatementDistributionMessage::BackedCandidateManifest( + manifest, + )) .into(), )) .await; } let ack_peers_v2 = filter_by_peer_version(&ack_peers, ValidationVersion::V2.into()); - let ack_peers_vstaging = filter_by_peer_version(&ack_peers, ValidationVersion::VStaging.into()); + let ack_peers_v3 = filter_by_peer_version(&ack_peers, ValidationVersion::V3.into()); if !ack_peers_v2.is_empty() { gum::debug!( target: LOG_TARGET, @@ -1948,22 +1946,20 @@ async fn provide_candidate_to_grid( .await; } - if !ack_peers_vstaging.is_empty() { + if !ack_peers_v3.is_empty() { gum::debug!( target: LOG_TARGET, ?candidate_hash, local_validator = ?per_session.local_validator, - n_peers = ack_peers_vstaging.len(), - "Sending acknowledgement to vstaging peers" + n_peers = ack_peers_v3.len(), + "Sending acknowledgement to v3 peers" ); ctx.send_message(NetworkBridgeTxMessage::SendValidationMessage( - ack_peers_vstaging, - Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::BackedCandidateKnown( - acknowledgement, - ), - ) + ack_peers_v3, + Versioned::V3(protocol_v3::StatementDistributionMessage::BackedCandidateKnown( + acknowledgement, + )) .into(), )) .await; @@ -2293,8 +2289,8 @@ fn post_acknowledgement_statement_messages( ) .into(), )), - ValidationVersion::VStaging => messages.push(Versioned::VStaging( - protocol_vstaging::StatementDistributionMessage::Statement( + ValidationVersion::V3 => messages.push(Versioned::V3( + protocol_v3::StatementDistributionMessage::Statement( relay_parent, statement.as_unchecked().clone(), ) @@ -2441,9 +2437,9 @@ fn acknowledgement_and_statement_messages( let mut messages = match peer.1 { ValidationVersion::V2 => vec![(vec![peer.0], msg_v2.into())], - ValidationVersion::VStaging => vec![( + ValidationVersion::V3 => vec![( vec![peer.0], - Versioned::VStaging(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( + Versioned::V3(protocol_v2::StatementDistributionMessage::BackedCandidateKnown( acknowledgement, )) .into(), diff --git a/polkadot/node/network/statement-distribution/src/v2/tests/grid.rs b/polkadot/node/network/statement-distribution/src/v2/tests/grid.rs index 116116659cb1..aa1a473b833f 100644 --- a/polkadot/node/network/statement-distribution/src/v2/tests/grid.rs +++ b/polkadot/node/network/statement-distribution/src/v2/tests/grid.rs @@ -2830,7 +2830,7 @@ fn inactive_local_participates_in_grid() { send_peer_message( &mut overseer, peer_a.clone(), - protocol_vstaging::StatementDistributionMessage::BackedCandidateManifest(manifest), + protocol_v3::StatementDistributionMessage::BackedCandidateManifest(manifest), ) .await; From 871e9cfd63dff3ee638210db46a33af41ff005ed Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Thu, 7 Dec 2023 18:39:51 +0200 Subject: [PATCH 80/89] Remove network-protocol-staging Signed-off-by: Alexandru Gheorghe --- polkadot/Cargo.toml | 1 - polkadot/cli/Cargo.toml | 4 +--- polkadot/node/network/protocol/Cargo.toml | 3 --- polkadot/node/network/protocol/src/peer_set.rs | 7 ------- polkadot/node/service/Cargo.toml | 6 +----- .../parachain/test-parachains/undying/collator/Cargo.toml | 3 --- 6 files changed, 2 insertions(+), 22 deletions(-) diff --git a/polkadot/Cargo.toml b/polkadot/Cargo.toml index c0227823c6b2..e5fe654f31fe 100644 --- a/polkadot/Cargo.toml +++ b/polkadot/Cargo.toml @@ -64,7 +64,6 @@ jemalloc-allocator = [ "polkadot-node-core-pvf/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator", ] -network-protocol-staging = ["polkadot-cli/network-protocol-staging"] # Enables timeout-based tests supposed to be run only in CI environment as they may be flaky diff --git a/polkadot/cli/Cargo.toml b/polkadot/cli/Cargo.toml index 72b2a18f36b3..3f877923923b 100644 --- a/polkadot/cli/Cargo.toml +++ b/polkadot/cli/Cargo.toml @@ -74,6 +74,4 @@ malus = ["full-node", "service/malus"] runtime-metrics = [ "polkadot-node-metrics/runtime-metrics", "service/runtime-metrics", -] - -network-protocol-staging = ["service/network-protocol-staging"] +] \ No newline at end of file diff --git a/polkadot/node/network/protocol/Cargo.toml b/polkadot/node/network/protocol/Cargo.toml index c33b9eae3252..379334ded24a 100644 --- a/polkadot/node/network/protocol/Cargo.toml +++ b/polkadot/node/network/protocol/Cargo.toml @@ -27,6 +27,3 @@ bitvec = "1" [dev-dependencies] rand_chacha = "0.3.1" - -[features] -network-protocol-staging = [] diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index fb2e2d77f6ff..3a3c3edbbd0b 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -127,13 +127,6 @@ impl PeerSet { /// Networking layer relies on `get_main_version()` being the version /// of the main protocol name reported by [`PeerSetProtocolNames::get_main_name()`]. pub fn get_main_version(self) -> ProtocolVersion { - #[cfg(not(feature = "network-protocol-staging"))] - match self { - PeerSet::Validation => ValidationVersion::V2.into(), - PeerSet::Collation => CollationVersion::V2.into(), - } - - #[cfg(feature = "network-protocol-staging")] match self { PeerSet::Validation => ValidationVersion::V3.into(), PeerSet::Collation => CollationVersion::V2.into(), diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 448ab605aa92..23a7bace30eb 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -224,8 +224,4 @@ runtime-metrics = [ "polkadot-runtime-parachains/runtime-metrics", "rococo-runtime?/runtime-metrics", "westend-runtime?/runtime-metrics", -] - -network-protocol-staging = [ - "polkadot-node-network-protocol/network-protocol-staging", -] +] \ No newline at end of file diff --git a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml index 0de349eac011..fac05dada74e 100644 --- a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml @@ -39,6 +39,3 @@ sc-service = { path = "../../../../../substrate/client/service" } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } tokio = { version = "1.24.2", features = ["macros"] } - -[features] -network-protocol-staging = ["polkadot-cli/network-protocol-staging"] From a364b6495a76fcdb405926841445b708b20dd5d7 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 8 Dec 2023 10:41:26 +0200 Subject: [PATCH 81/89] Rename zombienet Signed-off-by: Alexandru Gheorghe --- .gitlab/pipeline/zombienet/polkadot.yml | 12 ++++++------ ...ing.toml => 0009-approval-voting-coalescing.toml} | 0 ...g.zndsl => 0009-approval-voting-coalescing.zndsl} | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename polkadot/zombienet_tests/functional/{0007-approval-voting-coalescing.toml => 0009-approval-voting-coalescing.toml} (100%) rename polkadot/zombienet_tests/functional/{0007-approval-voting-coalescing.zndsl => 0009-approval-voting-coalescing.zndsl} (96%) diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 9a5a6d671cbb..356abaa93cdd 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -115,29 +115,29 @@ zombienet-polkadot-functional-0006-parachains-max-tranche0: --local-dir="${LOCAL_DIR}/functional" --test="0006-parachains-max-tranche0.zndsl" -zombienet-polkadot-functional-0007-approval-voting-coalescing: +zombienet-polkadot-functional-0007-dispute-freshly-finalized: extends: - .zombienet-polkadot-common script: - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="${LOCAL_DIR}/functional" - --test="0007-approval-voting-coalescing.zndsl" + --test="0007-dispute-freshly-finalized.zndsl" -zombienet-polkadot-functional-0007-dispute-freshly-finalized: +zombienet-polkadot-functional-0008-dispute-old-finalized: extends: - .zombienet-polkadot-common script: - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="${LOCAL_DIR}/functional" - --test="0007-dispute-freshly-finalized.zndsl" + --test="0008-dispute-old-finalized.zndsl" -zombienet-polkadot-functional-0008-dispute-old-finalized: +zombienet-polkadot-functional-0009-approval-voting-coalescing: extends: - .zombienet-polkadot-common script: - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="${LOCAL_DIR}/functional" - --test="0008-dispute-old-finalized.zndsl" + --test="0009-approval-voting-coalescing.zndsl" zombienet-polkadot-smoke-0001-parachains-smoke-test: extends: diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml b/polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.toml similarity index 100% rename from polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.toml rename to polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.toml diff --git a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl b/polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.zndsl similarity index 96% rename from polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl rename to polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.zndsl index 43b2c5e13a80..1fc4f6784460 100644 --- a/polkadot/zombienet_tests/functional/0007-approval-voting-coalescing.zndsl +++ b/polkadot/zombienet_tests/functional/0009-approval-voting-coalescing.zndsl @@ -1,5 +1,5 @@ Description: Approval voting coalescing does not lag finality -Network: ./0007-approval-voting-coalescing.toml +Network: ./0009-approval-voting-coalescing.toml Creds: config # Check authority status. From de7b5c006eead38c8694e4ce09c754c75b0a9e82 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 8 Dec 2023 15:46:02 +0200 Subject: [PATCH 82/89] Add migration v11 HostConfiguration, missed during rebasing Signed-off-by: Alexandru Gheorghe --- polkadot/runtime/rococo/src/lib.rs | 1 + polkadot/runtime/westend/src/lib.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 6d6e54f1a97e..5f918119ca6d 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1631,6 +1631,7 @@ pub mod migrations { // Remove `im-online` pallet on-chain storage frame_support::migrations::RemovePallet::DbWeight>, + parachains_configuration::migration::v11::MigrateToV11, ); } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index b20e164fedf7..4d2807b45b7a 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1648,6 +1648,7 @@ pub mod migrations { ImOnlinePalletName, ::DbWeight, >, + parachains_configuration::migration::v11::MigrateToV11, ); } From 5cf522aac9db8658722663042f57edeef8ba1652 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Fri, 8 Dec 2023 17:22:12 +0200 Subject: [PATCH 83/89] Fix network_protocol_versioning_subsystem_msg Signed-off-by: Alexandru Gheorghe --- polkadot/node/network/bridge/src/rx/tests.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/polkadot/node/network/bridge/src/rx/tests.rs b/polkadot/node/network/bridge/src/rx/tests.rs index ad503ee21f53..6847b8a7e24d 100644 --- a/polkadot/node/network/bridge/src/rx/tests.rs +++ b/polkadot/node/network/bridge/src/rx/tests.rs @@ -1469,7 +1469,7 @@ fn network_protocol_versioning_subsystem_msg() { NetworkBridgeEvent::PeerConnected( peer, ObservedRole::Full, - ValidationVersion::V2.into(), + ValidationVersion::V3.into(), None, ), &mut virtual_overseer, @@ -1484,9 +1484,9 @@ fn network_protocol_versioning_subsystem_msg() { } let approval_distribution_message = - protocol_v2::ApprovalDistributionMessage::Approvals(Vec::new()); + protocol_v3::ApprovalDistributionMessage::Approvals(Vec::new()); - let msg = protocol_v2::ValidationProtocol::ApprovalDistribution( + let msg = protocol_v3::ValidationProtocol::ApprovalDistribution( approval_distribution_message.clone(), ); @@ -1502,7 +1502,7 @@ fn network_protocol_versioning_subsystem_msg() { virtual_overseer.recv().await, AllMessages::ApprovalDistribution( ApprovalDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage(p, Versioned::V2(m)) + NetworkBridgeEvent::PeerMessage(p, Versioned::V3(m)) ) ) => { assert_eq!(p, peer); @@ -1536,7 +1536,7 @@ fn network_protocol_versioning_subsystem_msg() { virtual_overseer.recv().await, AllMessages::StatementDistribution( StatementDistributionMessage::NetworkBridgeUpdate( - NetworkBridgeEvent::PeerMessage(p, Versioned::V2(m)) + NetworkBridgeEvent::PeerMessage(p, Versioned::V3(m)) ) ) => { assert_eq!(p, peer); From 83a7325f2ee8d26dfd31810fe0b90f33fe89a3d7 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 16:19:56 +0200 Subject: [PATCH 84/89] Fix GetApprovalSignatures ... discovered during benchmarking Signed-off-by: Alexandru Gheorghe --- .../network/approval-distribution/src/lib.rs | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/polkadot/node/network/approval-distribution/src/lib.rs b/polkadot/node/network/approval-distribution/src/lib.rs index 08a5e57a9fb1..d520febaef51 100644 --- a/polkadot/node/network/approval-distribution/src/lib.rs +++ b/polkadot/node/network/approval-distribution/src/lib.rs @@ -599,17 +599,29 @@ impl BlockEntry { &self, candidate_index: CandidateIndex, ) -> Vec { - let result: Option> = - self.candidates.get(candidate_index as usize).map(|candidate_entry| { - candidate_entry - .assignments - .iter() - .filter_map(|(validator, assignment_bitfield)| { - self.approval_entries.get(&(*validator, assignment_bitfield.clone())) - }) - .flat_map(|approval_entry| approval_entry.approvals.clone().into_iter()) - .collect() - }); + let result: Option< + HashMap<(ValidatorIndex, CandidateBitfield), IndirectSignedApprovalVoteV2>, + > = self.candidates.get(candidate_index as usize).map(|candidate_entry| { + candidate_entry + .assignments + .iter() + .filter_map(|(validator, assignment_bitfield)| { + self.approval_entries.get(&(*validator, assignment_bitfield.clone())) + }) + .flat_map(|approval_entry| { + approval_entry + .approvals + .clone() + .into_iter() + .filter(|(approved_candidates, _)| { + approved_candidates.bit_at(candidate_index.as_bit_index()) + }) + .map(|(approved_candidates, vote)| { + ((approval_entry.validator_index, approved_candidates), vote) + }) + }) + .collect() + }); result.map(|result| result.into_values().collect_vec()).unwrap_or_default() } From 4d33c9d632fe537a257c1458df4c7c5c73357760 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 16:35:09 +0200 Subject: [PATCH 85/89] Fix formatting issues Signed-off-by: Alexandru Gheorghe --- polkadot/cli/Cargo.toml | 2 +- polkadot/node/service/Cargo.toml | 2 +- polkadot/roadmap/implementers-guide/src/protocol-approval.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/cli/Cargo.toml b/polkadot/cli/Cargo.toml index 3f877923923b..5f20d99c2f64 100644 --- a/polkadot/cli/Cargo.toml +++ b/polkadot/cli/Cargo.toml @@ -74,4 +74,4 @@ malus = ["full-node", "service/malus"] runtime-metrics = [ "polkadot-node-metrics/runtime-metrics", "service/runtime-metrics", -] \ No newline at end of file +] diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 23a7bace30eb..81eff49ee305 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -224,4 +224,4 @@ runtime-metrics = [ "polkadot-runtime-parachains/runtime-metrics", "rococo-runtime?/runtime-metrics", "westend-runtime?/runtime-metrics", -] \ No newline at end of file +] diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index ebe4894f0b3e..b6aa16646ad2 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -301,9 +301,9 @@ To reduce the necessary network bandwidth and cpu time when a validator has more doing our best effort to send a single message that approves all available candidates with a single signature. The implemented heuristic, is that each time we are ready to create a signature and send a vote for a candidate we delay sending it until one of three things happen: -- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we have already checked and we are +- We gathered a maximum of `MAX_APPROVAL_COALESCE_COUNT` candidates that we have already checked and we are ready to sign approval for. -- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since checking oldest candidate and we were ready to sign +- `MAX_APPROVAL_COALESCE_WAIT_TICKS` have passed since checking oldest candidate and we were ready to sign and send the approval message. - We are already in the last third of the no-show period in order to avoid creating accidental no-shows, which in turn might trigger other assignments. From 30950d2abda45d8486adec5d68d24c5ac3e11fc9 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 16:40:37 +0200 Subject: [PATCH 86/89] Fixup cargo fmt Signed-off-by: Alexandru Gheorghe --- .../relay-chain-rpc-interface/src/rpc_client.rs | 3 +-- .../approval-voting/src/approval_db/v2/tests.rs | 2 +- .../approval-voting/src/approval_db/v3/tests.rs | 2 +- polkadot/node/subsystem-types/src/messages.rs | 17 +++++++++-------- polkadot/runtime/parachains/src/disputes.rs | 6 ++++-- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs index 205a5717f7a3..c64fff77a29f 100644 --- a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs +++ b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs @@ -32,8 +32,7 @@ use cumulus_primitives_core::{ relay_chain::{ async_backing::{AsyncBackingParams, BackingState}, slashing, - vstaging::ApprovalVotingParams, - vstaging::NodeFeatures, + vstaging::{ApprovalVotingParams, NodeFeatures}, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash as RelayHash, Header as RelayHeader, InboundHrmpMessage, OccupiedCoreAssumption, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs index 3dbda3639d28..6021b44c2765 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v2/tests.rs @@ -18,7 +18,7 @@ use crate::{ approval_db::{ - common::{DbBackend, StoredBlockRange, *, migration_helpers::make_bitvec}, + common::{migration_helpers::make_bitvec, DbBackend, StoredBlockRange, *}, v2::*, v3::{load_block_entry_v2, load_candidate_entry_v2}, }, diff --git a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs index 29b3373e3f25..08c65461bca8 100644 --- a/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs +++ b/polkadot/node/core/approval-voting/src/approval_db/v3/tests.rs @@ -18,7 +18,7 @@ use crate::{ approval_db::{ - common::{DbBackend, StoredBlockRange, *, migration_helpers::make_bitvec}, + common::{migration_helpers::make_bitvec, DbBackend, StoredBlockRange, *}, v3::*, }, backend::{Backend, OverlayedBackend}, diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 8d5d3f055a03..c7675c84b91c 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -42,14 +42,15 @@ use polkadot_node_primitives::{ ValidationResult, }; use polkadot_primitives::{ - async_backing, slashing, vstaging::{NodeFeatures, ApprovalVotingParams}, AuthorityDiscoveryId, BackedCandidate, - BlockNumber, CandidateEvent, CandidateHash, CandidateIndex, CandidateReceipt, CollatorId, - CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupIndex, - GroupRotationInfo, Hash, Header as BlockHeader, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, MultiDisputeStatementSet, OccupiedCoreAssumption, PersistedValidationData, - PvfCheckStatement, PvfExecKind, SessionIndex, SessionInfo, SignedAvailabilityBitfield, - SignedAvailabilityBitfields, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - ValidatorSignature, + async_backing, slashing, + vstaging::{ApprovalVotingParams, NodeFeatures}, + AuthorityDiscoveryId, BackedCandidate, BlockNumber, CandidateEvent, CandidateHash, + CandidateIndex, CandidateReceipt, CollatorId, CommittedCandidateReceipt, CoreState, + DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Header as BlockHeader, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, MultiDisputeStatementSet, + OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, PvfExecKind, SessionIndex, + SessionInfo, SignedAvailabilityBitfield, SignedAvailabilityBitfields, ValidationCode, + ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use polkadot_statement_table::v2::Misbehavior; use std::{ diff --git a/polkadot/runtime/parachains/src/disputes.rs b/polkadot/runtime/parachains/src/disputes.rs index ca275cf6442c..c2383dad3053 100644 --- a/polkadot/runtime/parachains/src/disputes.rs +++ b/polkadot/runtime/parachains/src/disputes.rs @@ -1017,8 +1017,10 @@ impl Pallet { set.session, statement, signature, - // This is here to prevent malicious nodes of generating `ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates` - // before that is enabled, via setting `max_approval_coalesce_count` in the parachain host config. + // This is here to prevent malicious nodes of generating + // `ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates` before that + // is enabled, via setting `max_approval_coalesce_count` in the parachain host + // config. config.approval_voting_params.max_approval_coalesce_count > 1, ) { log::warn!("Failed to check dispute signature"); From 593b82ab003288e57c8b8c83bf6d57a72c8fd50c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 17:13:01 +0200 Subject: [PATCH 87/89] Fix some logging messed during rebase Signed-off-by: Alexandru Gheorghe --- .../runtime/parachains/src/configuration/migration/v11.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/runtime/parachains/src/configuration/migration/v11.rs b/polkadot/runtime/parachains/src/configuration/migration/v11.rs index 053386be6e46..b7dec7070f93 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v11.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v11.rs @@ -71,22 +71,22 @@ pub struct UncheckedMigrateToV11(sp_std::marker::PhantomData); impl OnRuntimeUpgrade for UncheckedMigrateToV11 { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { - log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV10"); + log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV11"); Ok(Vec::new()) } fn on_runtime_upgrade() -> Weight { - log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 started"); + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV11 started"); let weight_consumed = migrate_to_v11::(); - log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV10 executed successfully"); + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV11 executed successfully"); weight_consumed } #[cfg(feature = "try-runtime")] fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { - log::trace!(target: crate::configuration::LOG_TARGET, "Running post_upgrade() for HostConfiguration MigrateToV10"); + log::trace!(target: crate::configuration::LOG_TARGET, "Running post_upgrade() for HostConfiguration MigrateToV11"); ensure!( StorageVersion::get::>() >= 11, "Storage version should be >= 11 after the migration" From f8f03e572a589156d9ac8a7500aacac3e61bcd04 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Mon, 11 Dec 2023 17:39:06 +0200 Subject: [PATCH 88/89] Add prdoc Signed-off-by: Alexandru Gheorghe --- prdoc/pr_1191.prdoc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 prdoc/pr_1191.prdoc diff --git a/prdoc/pr_1191.prdoc b/prdoc/pr_1191.prdoc new file mode 100644 index 000000000000..26626731be46 --- /dev/null +++ b/prdoc/pr_1191.prdoc @@ -0,0 +1,21 @@ +title: Approve multiple candidates with a single signature + +doc: + - audience: Node Operator + description: | + Changed approval-voting, approval-distribution to approve multiple candidate with a single message, it adds: + * A new parachains_db version. + * A new validation protocol to support the new message types. + The new logic will be disabled and will be enabled at a later date after all validators have upgraded. + +migrations: + db: + - name: Parachains database change from v4 to v5. + description: | + Approval-voting column format has been updated with several new fields. All existing data will be automatically + be migrated to the new values. + +crates: + - name: "polkadot" + +host_functions: [] From 8099c16e7b2c9a24be929f51083c6a31d770528a Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe Date: Tue, 12 Dec 2023 13:51:03 +0200 Subject: [PATCH 89/89] Fixup 0002-upgrade-node failures V2 was not put into the list of fallbacks for the validation protocol, so the test wrongly fall-backed on v1. Signed-off-by: Alexandru Gheorghe --- .../node/network/protocol/src/peer_set.rs | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index 3a3c3edbbd0b..cb329607ad61 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -73,7 +73,11 @@ impl PeerSet { // Networking layer relies on `get_main_name()` being the main name of the protocol // for peersets and connection management. let protocol = peerset_protocol_names.get_main_name(self); - let fallback_names = PeerSetProtocolNames::get_fallback_names(self); + let fallback_names = PeerSetProtocolNames::get_fallback_names( + self, + &peerset_protocol_names.genesis_hash, + peerset_protocol_names.fork_id.as_deref(), + ); let max_notification_size = self.get_max_notification_size(is_authority); match self { @@ -293,6 +297,8 @@ impl From for ProtocolVersion { pub struct PeerSetProtocolNames { protocols: HashMap, names: HashMap<(PeerSet, ProtocolVersion), ProtocolName>, + genesis_hash: Hash, + fork_id: Option, } impl PeerSetProtocolNames { @@ -327,7 +333,7 @@ impl PeerSetProtocolNames { } Self::register_legacy_protocol(&mut protocols, protocol); } - Self { protocols, names } + Self { protocols, names, genesis_hash, fork_id: fork_id.map(|fork_id| fork_id.into()) } } /// Helper function to register main protocol. @@ -431,9 +437,30 @@ impl PeerSetProtocolNames { } /// Get the protocol fallback names. Currently only holds the legacy name - /// for `LEGACY_PROTOCOL_VERSION` = 1. - fn get_fallback_names(protocol: PeerSet) -> Vec { - std::iter::once(Self::get_legacy_name(protocol)).collect() + /// for `LEGACY_PROTOCOL_VERSION` = 1 and v2 for validation. + fn get_fallback_names( + protocol: PeerSet, + genesis_hash: &Hash, + fork_id: Option<&str>, + ) -> Vec { + let mut fallbacks = vec![Self::get_legacy_name(protocol)]; + match protocol { + PeerSet::Validation => { + // Fallbacks are tried one by one, till one matches so push v2 at the top, so + // that it is used ahead of the legacy one(v1). + fallbacks.insert( + 0, + Self::generate_name( + genesis_hash, + fork_id, + protocol, + ValidationVersion::V2.into(), + ), + ) + }, + PeerSet::Collation => {}, + }; + fallbacks } }