Merge 'master' and resolve conflicts

* master:
  disputes pallet: Remove spam slots (#6345)
  Use `polkadot-node-metrics` where possible (#6484)
  Unlimited Proof size per block (#6085)
  [ci] Remove check-transaction-versions job (#6509)
  Update serde because substrate->trybuild needs a newer version (#6505)
  upgrade libp2p to 0.50.0 (#6500)
  av-store: write meta for unknown finalized blocks (#6452)

node/core/av-store/src/lib.rs

@@ -570,6 +570,14 @@ async fn run_iteration<Context>(
 				FromOrchestra::Signal(OverseerSignal::BlockFinalized(hash, number)) => {
 					let _timer = subsystem.metrics.time_process_block_finalized();
 
+					if !subsystem.known_blocks.is_known(&hash) {
+						// If we haven't processed this block yet,
+						// make sure we write the metadata about the
+						// candidates backed in this finalized block.
+						// Otherwise, we won't be able to store our chunk
+						// for these candidates.
+						process_block_activated(ctx, subsystem, hash).await?;
+					}
 					subsystem.finalized_number = Some(number);
 					subsystem.known_blocks.prune_finalized(number);
 					process_block_finalized(

node/core/av-store/src/tests.rs

@@ -732,8 +732,49 @@ fn we_dont_miss_anything_if_import_notifications_are_missed() {
 	let test_state = TestState::default();
 
 	test_harness(test_state.clone(), store.clone(), |mut virtual_overseer| async move {
-		overseer_signal(&mut virtual_overseer, OverseerSignal::BlockFinalized(Hash::zero(), 1))
-			.await;
+		let block_hash = Hash::repeat_byte(1);
+		overseer_signal(&mut virtual_overseer, OverseerSignal::BlockFinalized(block_hash, 1)).await;
+
+		let header = Header {
+			parent_hash: Hash::repeat_byte(0),
+			number: 1,
+			state_root: Hash::zero(),
+			extrinsics_root: Hash::zero(),
+			digest: Default::default(),
+		};
+
+		assert_matches!(
+			overseer_recv(&mut virtual_overseer).await,
+			AllMessages::ChainApi(ChainApiMessage::BlockHeader(
+				relay_parent,
+				tx,
+			)) => {
+				assert_eq!(relay_parent, block_hash);
+				tx.send(Ok(Some(header))).unwrap();
+			}
+		);
+
+		assert_matches!(
+			overseer_recv(&mut virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::CandidateEvents(tx),
+			)) => {
+				assert_eq!(relay_parent, block_hash);
+				tx.send(Ok(Vec::new())).unwrap();
+			}
+		);
+
+		assert_matches!(
+			overseer_recv(&mut virtual_overseer).await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				relay_parent,
+				RuntimeApiRequest::Validators(tx),
+			)) => {
+				assert_eq!(relay_parent, Hash::zero());
+				tx.send(Ok(Vec::new())).unwrap();
+			}
+		);
 
 		let header = Header {
 			parent_hash: Hash::repeat_byte(3),

node/core/candidate-validation/Cargo.toml

@@ -16,8 +16,8 @@ parity-scale-codec = { version = "3.1.5", default-features = false, features = [
 polkadot-primitives = { path = "../../../primitives" }
 polkadot-parachain = { path = "../../../parachain" }
 polkadot-node-primitives = { path = "../../primitives" }
-polkadot-node-subsystem = {path = "../../subsystem" }
-polkadot-node-subsystem-util = { path = "../../subsystem-util" }
+polkadot-node-subsystem = { path = "../../subsystem" }
+polkadot-node-metrics = { path = "../../metrics" }
 
 [target.'cfg(not(any(target_os = "android", target_os = "unknown")))'.dependencies]
 polkadot-node-core-pvf = { path = "../pvf" }
@@ -27,5 +27,6 @@ sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master
 futures = { version = "0.3.21", features = ["thread-pool"] }
 assert_matches = "1.4.0"
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
+polkadot-node-subsystem-util = { path = "../../subsystem-util" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }

node/core/candidate-validation/src/metrics.rs

@@ -15,7 +15,7 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 use super::{ValidationFailed, ValidationResult};
-use polkadot_node_subsystem_util::metrics::{self, prometheus};
+use polkadot_node_metrics::metrics::{self, prometheus};
 
 #[derive(Clone)]
 pub(crate) struct MetricsInner {

node/core/chain-api/Cargo.toml

@@ -9,8 +9,8 @@ futures = "0.3.21"
 gum = { package = "tracing-gum", path = "../../gum" }
 sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-primitives = { path = "../../../primitives" }
+polkadot-node-metrics = { path = "../../metrics" }
 polkadot-node-subsystem = {path = "../../subsystem" }
-polkadot-node-subsystem-util = { path = "../../subsystem-util" }
 sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
 

node/core/chain-api/src/metrics.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use polkadot_node_subsystem_util::metrics::{self, prometheus};
+use polkadot_node_metrics::metrics::{self, prometheus};
 
 #[derive(Clone)]
 pub(crate) struct MetricsInner {

node/core/runtime-api/Cargo.toml

@@ -12,9 +12,9 @@ lru = "0.8"
 sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 polkadot-primitives = { path = "../../../primitives" }
+polkadot-node-metrics = { path = "../../metrics" }
 polkadot-node-subsystem = { path = "../../subsystem" }
 polkadot-node-subsystem-types = { path = "../../subsystem-types" }
-polkadot-node-subsystem-util = { path = "../../subsystem-util" }
 
 [dev-dependencies]
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }

node/core/runtime-api/src/metrics.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use polkadot_node_subsystem_util::metrics::{self, prometheus};
+use polkadot_node_metrics::metrics::{self, prometheus};
 
 #[derive(Clone)]
 pub(crate) struct MetricsInner {

node/network/approval-distribution/Cargo.toml

@@ -5,10 +5,10 @@ authors.workspace = true
 edition.workspace = true
 
 [dependencies]
-polkadot-node-primitives = { path = "../../primitives" }
+polkadot-node-metrics = { path = "../../metrics" }
 polkadot-node-network-protocol = { path = "../protocol" }
+polkadot-node-primitives = { path = "../../primitives" }
 polkadot-node-subsystem = { path = "../../subsystem" }
-polkadot-node-subsystem-util = { path = "../../subsystem-util" }
 polkadot-primitives = { path = "../../../primitives" }
 rand = "0.8"
 

node/network/approval-distribution/src/metrics.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use polkadot_node_subsystem_util::metrics::{prometheus, Metrics as MetricsTrait};
+use polkadot_node_metrics::metrics::{prometheus, Metrics as MetricsTrait};
 
 /// Approval Distribution metrics.
 #[derive(Default, Clone)]

node/network/bridge/Cargo.toml

@@ -14,10 +14,10 @@ parity-scale-codec = { version = "3.1.5", default-features = false, features = [
 sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-network-common = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
+polkadot-node-metrics = { path = "../../metrics"}
+polkadot-node-network-protocol = { path = "../protocol" }
 polkadot-node-subsystem = {path = "../../subsystem" }
 polkadot-overseer = { path = "../../overseer" }
-polkadot-node-network-protocol = { path = "../protocol" }
-polkadot-node-subsystem-util = { path = "../../subsystem-util"}
 parking_lot = "0.12.0"
 bytes = "1"
 fatality = "0.0.6"
@@ -26,6 +26,7 @@ thiserror = "1"
 [dev-dependencies]
 assert_matches = "1.4.0"
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
+polkadot-node-subsystem-util = { path = "../../subsystem-util"}
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
 futures-timer = "3"

node/network/bridge/src/metrics.rs

@@ -15,7 +15,7 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 use super::{PeerSet, ProtocolVersion};
-use polkadot_node_subsystem_util::metrics::{self, prometheus};
+use polkadot_node_metrics::metrics::{self, prometheus};
 
 /// Metrics for the network bridge.
 #[derive(Clone, Default)]

roadmap/implementers-guide/src/runtime/disputes.md

@@ -37,12 +37,6 @@ Disputes: double_map (SessionIndex, CandidateHash) -> Option<DisputeState>,
 // All included blocks on the chain, as well as the block number in this chain that
 // should be reverted back to if the candidate is disputed and determined to be invalid.
 Included: double_map (SessionIndex, CandidateHash) -> Option<BlockNumber>,
-// Maps session indices to a vector indicating the number of potentially-spam disputes 
-// each validator is participating in. Potentially-spam disputes are remote disputes which have
-// fewer than `byzantine_threshold + 1` validators.
-//
-// The i'th entry of the vector corresponds to the i'th validator in the session.
-SpamSlots: map SessionIndex -> Option<Vec<u32>>,
 // Whether the chain is frozen or not. Starts as `None`. When this is `Some`,
 // the chain will not accept any new parachain blocks for backing or inclusion,
 // and its value indicates the last valid block number in the chain.
@@ -55,50 +49,45 @@ Frozen: Option<BlockNumber>,
 ## Session Change
 
 1. If the current session is not greater than `config.dispute_period + 1`, nothing to do here.
-1. Set `pruning_target = current_session - config.dispute_period - 1`. We add the extra `1` because we want to keep things for `config.dispute_period` _full_ sessions. 
+1. Set `pruning_target = current_session - config.dispute_period - 1`. We add the extra `1` because we want to keep things for `config.dispute_period` _full_ sessions.
    The stuff at the end of the most recent session has been around for a little over 0 sessions, not a little over 1.
 1. If `LastPrunedSession` is `None`, then set `LastPrunedSession` to `Some(pruning_target)` and return.
-1. Otherwise, clear out all disputes, included candidates, and `SpamSlots` entries in the range `last_pruned..=pruning_target` and set `LastPrunedSession` to `Some(pruning_target)`.
+2. Otherwise, clear out all disputes and included candidates entries in the range `last_pruned..=pruning_target` and set `LastPrunedSession` to `Some(pruning_target)`.
 
 ## Block Initialization
 
-1. Iterate through all disputes. If any have not concluded and started more than `config.dispute_conclusion_by_timeout_period` blocks ago, set them to `Concluded` and mildly punish all validators associated, as they have failed to distribute available data. If the `Included` map does not contain the candidate and there are fewer than `byzantine_threshold + 1` participating validators, reduce `SpamSlots` for all participating validators.
+1. Iterate through all disputes. If any have not concluded and started more than `config.dispute_conclusion_by_timeout_period` blocks ago, set them to `Concluded` and mildly punish all validators associated, as they have failed to distribute available data.
 
 ## Routines
 
 * `filter_multi_dispute_data(MultiDisputeStatementSet) -> MultiDisputeStatementSet`:
   1. Takes a `MultiDisputeStatementSet` and filters it down to a `MultiDisputeStatementSet`
     that satisfies all the criteria of `provide_multi_dispute_data`. That is, eliminating
-    ancient votes, votes which overwhelm the maximum amount of spam slots, and duplicates. 
-    This can be used by block authors to create the final submission in a block which is 
+    ancient votes, duplicates and unconfirmed disputes.
+    This can be used by block authors to create the final submission in a block which is
     guaranteed to pass the `provide_multi_dispute_data` checks.
 
 * `provide_multi_dispute_data(MultiDisputeStatementSet) -> Vec<(SessionIndex, Hash)>`:
   1. Pass on each dispute statement set to `provide_dispute_data`, propagating failure.
-  1. Return a list of all candidates who just had disputes initiated.
+  2. Return a list of all candidates who just had disputes initiated.
 
 * `provide_dispute_data(DisputeStatementSet) -> bool`: Provide data to an ongoing dispute or initiate a dispute.
-  1. All statements must be issued under the correct session for the correct candidate. 
+  1. All statements must be issued under the correct session for the correct candidate.
   1. `SessionInfo` is used to check statement signatures and this function should fail if any signatures are invalid.
   1. If there is no dispute under `Disputes`, create a new `DisputeState` with blank bitfields.
   1. If `concluded_at` is `Some`, and is `concluded_at + config.post_conclusion_acceptance_period < now`, return false.
-  1. If the overlap of the validators in the `DisputeStatementSet` and those already present in the `DisputeState` is fewer in number than `byzantine_threshold + 1` and the candidate is not present in the `Included` map
-      1. increment `SpamSlots` for each validator in the `DisputeStatementSet` which is not already in the `DisputeState`. Initialize the `SpamSlots` to a zeroed vector first, if necessary. do not increment `SpamSlots` if the candidate is local.
-      1. If the value for any spam slot exceeds `config.dispute_max_spam_slots`, return false.
-  1. If the overlap of the validators in the `DisputeStatementSet` and those already present in the `DisputeState` is at least `byzantine_threshold + 1`, the `DisputeState` has fewer than `byzantine_threshold + 1` validators, and the candidate is not present in the `Included` map, then decrease `SpamSlots` by 1 for each validator in the `DisputeState`.
-  1. Import all statements into the dispute. This should fail if any statements are duplicate or if the corresponding bit for the corresponding validator is set in the dispute already.
-  1. If `concluded_at` is `None`, reward all statements.
-  1. If `concluded_at` is `Some`, reward all statements slightly less.
-  1. If either side now has supermajority and did not previously, slash the other side. This may be both sides, and we support this possibility in code, but note that this requires validators to participate on both sides which has negative expected value. Set `concluded_at` to `Some(now)` if it was `None`.
-  1. If just concluded against the candidate and the `Included` map contains `(session, candidate)`: invoke `revert_and_freeze` with the stored block number.
-  1. Return true if just initiated, false otherwise.
+  2. Import all statements into the dispute. This should fail if any statements are duplicate or if the corresponding bit for the corresponding validator is set in the dispute already.
+  3. If `concluded_at` is `None`, reward all statements.
+  4. If `concluded_at` is `Some`, reward all statements slightly less.
+  5. If either side now has supermajority and did not previously, slash the other side. This may be both sides, and we support this possibility in code, but note that this requires validators to participate on both sides which has negative expected value. Set `concluded_at` to `Some(now)` if it was `None`.
+  6. If just concluded against the candidate and the `Included` map contains `(session, candidate)`: invoke `revert_and_freeze` with the stored block number.
+  7. Return true if just initiated, false otherwise.
 
 * `disputes() -> Vec<(SessionIndex, CandidateHash, DisputeState)>`: Get a list of all disputes and info about dispute state.
   1. Iterate over all disputes in `Disputes` and collect into a vector.
 
 * `note_included(SessionIndex, CandidateHash, included_in: BlockNumber)`:
   1. Add `(SessionIndex, CandidateHash)` to the `Included` map with `included_in - 1` as the value.
-  1. If there is a dispute under `(Sessionindex, CandidateHash)` with fewer than `byzantine_threshold + 1` participating validators, decrease `SpamSlots` by 1 for each validator in the `DisputeState`.
   1. If there is a dispute under `(SessionIndex, CandidateHash)` that has concluded against the candidate, invoke `revert_and_freeze` with the stored block number.
 
 * `concluded_invalid(SessionIndex, CandidateHash) -> bool`: Returns whether a candidate has already concluded a dispute in the negative.
@@ -111,3 +100,27 @@ Frozen: Option<BlockNumber>,
   1. If `is_frozen()` return.
   1. Set `Frozen` to `Some(BlockNumber)` to indicate a rollback to the block number.
   1. Issue a `Revert(BlockNumber + 1)` log to indicate a rollback of the block's child in the header chain, which is the same as a rollback to the block number.
+
+# Disputes filtering
+
+All disputes delivered to the runtime by the client are filtered before the actual import. In this context actual import
+means persisted in the runtime storage. The filtering has got two purposes:
+- Limit the amount of data saved onchain.
+- Prevent persisting malicious dispute data onchain.
+
+*Implementation note*: Filtering is performed in function `filter_dispute_data` from `Disputes` pallet.
+
+The filtering is performed on the whole statement set which is about to be imported onchain. The following filters are
+applied:
+1. Remove ancient disputes - if a dispute is concluded before the block number indicated in `OLDEST_ACCEPTED` parameter
+   it is removed from the set. `OLDEST_ACCEPTED` is a runtime configuration option.
+   *Implementation note*: `dispute_post_conclusion_acceptance_period` from
+   `HostConfiguration` is used in the current Polkadot/Kusama implementation.
+2. Remove votes from unknown validators. If there is a vote from a validator which wasn't an authority in the session
+   where the dispute was raised - they are removed. Please note that this step removes only single votes instead of
+   removing the whole dispute.
+3. Remove one sided disputes - if a dispute doesn't contain two opposing votes it is not imported onchain. This serves
+   as a measure not to import one sided disputes. A dispute is raised only if there are two opposing votes so if the
+   client is not sending them the dispute is a potential spam.
+4. Remove unconfirmed disputes - if a dispute contains less votes than the byzantine threshold it is removed. This is
+   also a spam precaution. A legitimate client will send only confirmed disputes to the runtime.

runtime/common/Cargo.toml

@@ -11,7 +11,7 @@ parity-scale-codec = { version = "3.1.5", default-features = false, features = [
 log = { version = "0.4.17", default-features = false }
 rustc-hex = { version = "2.1.0", default-features = false }
 scale-info = { version = "2.1.2", default-features = false, features = ["derive"] }
-serde = { version = "1.0.137", default-features = false }
+serde = { version = "1.0.139", default-features = false }
 serde_derive = { version = "1.0.117", optional = true }
 static_assertions = "1.1.0"
 

runtime/common/src/lib.rs

@@ -43,7 +43,7 @@ use frame_support::{
 	weights::{constants::WEIGHT_REF_TIME_PER_SECOND, Weight},
 };
 use frame_system::limits;
-use primitives::v2::{AssignmentId, Balance, BlockNumber, ValidatorId, MAX_POV_SIZE};
+use primitives::v2::{AssignmentId, Balance, BlockNumber, ValidatorId};
 use sp_runtime::{FixedPointNumber, Perbill, Perquintill};
 use static_assertions::const_assert;
 
@@ -69,9 +69,10 @@ pub const AVERAGE_ON_INITIALIZE_RATIO: Perbill = Perbill::from_percent(1);
 /// We allow `Normal` extrinsics to fill up the block up to 75%, the rest can be used
 /// by  Operational  extrinsics.
 pub const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75);
+/// We allow for 2 seconds of compute with a 6 second average block time.
 /// The storage proof size is not limited so far.
 pub const MAXIMUM_BLOCK_WEIGHT: Weight =
-	Weight::from_parts(WEIGHT_REF_TIME_PER_SECOND.saturating_mul(2), MAX_POV_SIZE as u64);
+	Weight::from_parts(WEIGHT_REF_TIME_PER_SECOND.saturating_mul(2), u64::MAX);
 
 const_assert!(NORMAL_DISPATCH_RATIO.deconstruct() >= AVERAGE_ON_INITIALIZE_RATIO.deconstruct());
 

runtime/kusama/Cargo.toml

@@ -11,7 +11,7 @@ parity-scale-codec = { version = "3.1.5", default-features = false, features = [
 scale-info = { version = "2.1.2", default-features = false, features = ["derive"] }
 log = { version = "0.4.17", default-features = false }
 rustc-hex = { version = "2.1.0", default-features = false }
-serde = { version = "1.0.137", default-features = false }
+serde = { version = "1.0.139", default-features = false }
 serde_derive = { version = "1.0.117", optional = true }
 static_assertions = "1.1.0"
 smallvec = "1.8.0"

runtime/kusama/src/lib.rs

@@ -1493,6 +1493,8 @@ pub type Migrations = (
 	>,
 	pallet_scheduler::migration::v4::CleanupAgendas<Runtime>,
 	pallet_staking::migrations::v13::MigrateToV13<Runtime>,
+	parachains_disputes::migration::v1::MigrateToV1<Runtime>,
+	parachains_configuration::migration::v4::MigrateToV4<Runtime>,
 );
 
 /// Unchecked extrinsic type as expected by this runtime.