Proposal: Flatten AllPallets and similar types (paritytech#11813)

* flratten AllPallets types

* feature flag it

* fix

* fix

* fmt

* remove todo

* Update frame/support/src/traits/metadata.rs

Co-authored-by: Bastian Köcher <[email protected]>

* Update frame/support/src/migrations.rs

Co-authored-by: Bastian Köcher <[email protected]>

* fix

* mark as deprecated

* add docs

* fix ui test?

* fmt

Co-authored-by: parity-processbot <>
Co-authored-by: Bastian Köcher <[email protected]>

frame/support/Cargo.toml

@@ -73,3 +73,7 @@ no-metadata-docs = ["frame-support-procedural/no-metadata-docs"]
 # By default some types have documentation, `full-metadata-docs` allows to add documentation to
 # more types in the metadata.
 full-metadata-docs = ["scale-info/docs"]
+# Generate impl-trait for tuples with the given number of tuples. Will be needed as the number of
+# pallets in a runtime grows. Does increase the compile time!
+tuples-96 = []
+tuples-128 = []

frame/support/procedural/src/construct_runtime/mod.rs

@@ -308,47 +308,26 @@ fn decl_all_pallets<'a>(
 		names.push(&pallet_declaration.name);
 	}
 
-	// Make nested tuple structure like:
-	// `((FirstPallet, (SecondPallet, ( ... , LastPallet) ... ))))`
-	// But ignore the system pallet.
-	let all_pallets_without_system = names
-		.iter()
-		.filter(|n| **n != SYSTEM_PALLET_NAME)
-		.rev()
-		.fold(TokenStream2::default(), |combined, name| quote!((#name, #combined)));
-
-	// Make nested tuple structure like:
-	// `((FirstPallet, (SecondPallet, ( ... , LastPallet) ... ))))`
-	let all_pallets_with_system = names
-		.iter()
-		.rev()
-		.fold(TokenStream2::default(), |combined, name| quote!((#name, #combined)));
-
-	// Make nested tuple structure like:
-	// `((LastPallet, (SecondLastPallet, ( ... , FirstPallet) ... ))))`
-	// But ignore the system pallet.
-	let all_pallets_without_system_reversed = names
-		.iter()
-		.filter(|n| **n != SYSTEM_PALLET_NAME)
-		.fold(TokenStream2::default(), |combined, name| quote!((#name, #combined)));
-
-	// Make nested tuple structure like:
-	// `((LastPallet, (SecondLastPallet, ( ... , FirstPallet) ... ))))`
-	let all_pallets_with_system_reversed = names
-		.iter()
-		.fold(TokenStream2::default(), |combined, name| quote!((#name, #combined)));
-
 	let system_pallet = match names.iter().find(|n| **n == SYSTEM_PALLET_NAME) {
 		Some(name) => name,
 		None =>
 			return syn::Error::new(
 				proc_macro2::Span::call_site(),
 				"`System` pallet declaration is missing. \
-				 Please add this line: `System: frame_system::{Pallet, Call, Storage, Config, Event<T>},`",
+					 Please add this line: `System: frame_system::{Pallet, Call, Storage, Config, Event<T>},`",
 			)
 			.into_compile_error(),
 	};
 
+	let names_without_system =
+		names.iter().filter(|n| **n != SYSTEM_PALLET_NAME).collect::<Vec<_>>();
+	let names_reversed = names.clone().into_iter().rev().collect::<Vec<_>>();
+	let names_without_system_reverse =
+		names_without_system.clone().into_iter().rev().collect::<Vec<_>>();
+	let names_reversed_with_system_first = std::iter::once(system_pallet)
+		.chain(names_without_system_reverse.clone().into_iter())
+		.collect::<Vec<_>>();
+
 	quote!(
 		#types
 
@@ -364,25 +343,28 @@ fn decl_all_pallets<'a>(
 		pub type AllPallets = AllPalletsWithSystem;
 
 		/// All pallets included in the runtime as a nested tuple of types.
-		pub type AllPalletsWithSystem = ( #all_pallets_with_system );
+		pub type AllPalletsWithSystem = ( #(#names),* );
 
 		/// All pallets included in the runtime as a nested tuple of types.
 		/// Excludes the System pallet.
-		pub type AllPalletsWithoutSystem = ( #all_pallets_without_system );
+		pub type AllPalletsWithoutSystem = ( #(#names_without_system),* );
 
 		/// All pallets included in the runtime as a nested tuple of types in reversed order.
 		/// Excludes the System pallet.
-		pub type AllPalletsWithoutSystemReversed = ( #all_pallets_without_system_reversed );
+		#[deprecated(note = "Using reverse pallet orders is deprecated. use only \
+		`AllPalletWithSystem or AllPalletsWithoutSystem`")]
+		pub type AllPalletsWithoutSystemReversed =( #(#names_without_system_reverse),* );
 
 		/// All pallets included in the runtime as a nested tuple of types in reversed order.
-		pub type AllPalletsWithSystemReversed = ( #all_pallets_with_system_reversed );
+		#[deprecated(note = "Using reverse pallet orders is deprecated. use only \
+		`AllPalletWithSystem or AllPalletsWithoutSystem`")]
+		pub type AllPalletsWithSystemReversed = ( #(#names_reversed),* );
 
 		/// All pallets included in the runtime as a nested tuple of types in reversed order.
 		/// With the system pallet first.
-		pub type AllPalletsReversedWithSystemFirst = (
-			#system_pallet,
-			AllPalletsWithoutSystemReversed
-		);
+		#[deprecated(note = "Using reverse pallet orders is deprecated. use only \
+		`AllPalletWithSystem or AllPalletsWithoutSystem`")]
+		pub type AllPalletsReversedWithSystemFirst = ( #(#names_reversed_with_system_first),* );
 	)
 }
 

frame/support/procedural/src/pallet/expand/pallet_struct.rs

@@ -240,17 +240,15 @@ pub fn expand_pallet_struct(def: &mut Def) -> proc_macro2::TokenStream {
 			#config_where_clause
 		{
 			fn count() -> usize { 1 }
-			fn accumulate(
-				acc: &mut #frame_support::sp_std::vec::Vec<#frame_support::traits::PalletInfoData>
-			) {
+			fn infos() -> #frame_support::sp_std::vec::Vec<#frame_support::traits::PalletInfoData> {
 				use #frame_support::traits::PalletInfoAccess;
 				let item = #frame_support::traits::PalletInfoData {
 					index: Self::index(),
 					name: Self::name(),
 					module_name: Self::module_name(),
 					crate_version: Self::crate_version(),
 				};
-				acc.push(item);
+				#frame_support::sp_std::vec![item]
 			}
 		}
 

frame/support/src/dispatch.rs

@@ -2207,15 +2207,15 @@ macro_rules! decl_module {
 			for $mod_type<$trait_instance $(, $instance)?> where $( $other_where_bounds )*
 		{
 			fn count() -> usize { 1 }
-			fn accumulate(acc: &mut $crate::sp_std::vec::Vec<$crate::traits::PalletInfoData>) {
+			fn infos() -> $crate::sp_std::vec::Vec<$crate::traits::PalletInfoData> {
 				use $crate::traits::PalletInfoAccess;
 				let item = $crate::traits::PalletInfoData {
 					index: Self::index(),
 					name: Self::name(),
 					module_name: Self::module_name(),
 					crate_version: Self::crate_version(),
 				};
-				acc.push(item);
+				vec![item]
 			}
 		}
 

frame/support/src/lib.rs

@@ -16,6 +16,16 @@
 // limitations under the License.
 
 //! Support code for the runtime.
+//!
+//! ## Note on Tuple Traits
+//!
+//! Many of the traits defined in [`traits`] have auto-implementations on tuples as well. Usually,
+//! the tuple is a function of number of pallets in the runtime. By default, the traits are
+//! implemented for tuples of up to 64 items.
+//
+// If you have more pallets in your runtime, or for any other reason need more, enabled `tuples-96`
+// or the `tuples-128` complication flag. Note that these features *will increase* the compilation
+// of this crate.
 
 #![cfg_attr(not(feature = "std"), no_std)]
 

frame/support/src/migrations.rs

@@ -19,6 +19,7 @@ use crate::{
 	traits::{GetStorageVersion, PalletInfoAccess},
 	weights::{RuntimeDbWeight, Weight},
 };
+use impl_trait_for_tuples::impl_for_tuples;
 
 /// Trait used by [`migrate_from_pallet_version_to_storage_version`] to do the actual migration.
 pub trait PalletVersionToStorageVersionHelper {
@@ -42,7 +43,9 @@ impl<T: GetStorageVersion + PalletInfoAccess> PalletVersionToStorageVersionHelpe
 	}
 }
 
-#[impl_trait_for_tuples::impl_for_tuples(30)]
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
 impl PalletVersionToStorageVersionHelper for T {
 	fn migrate(db_weight: &RuntimeDbWeight) -> Weight {
 		let mut weight: Weight = 0;

frame/support/src/traits.rs

@@ -50,7 +50,7 @@ mod error;
 pub use error::PalletError;
 
 mod filter;
-pub use filter::{ClearFilterGuard, FilterStack, FilterStackGuard, InstanceFilter, IntegrityTest};
+pub use filter::{ClearFilterGuard, FilterStack, FilterStackGuard, InstanceFilter};
 
 mod misc;
 pub use misc::{
@@ -81,7 +81,8 @@ mod hooks;
 #[cfg(feature = "std")]
 pub use hooks::GenesisBuild;
 pub use hooks::{
-	Hooks, OnFinalize, OnGenesis, OnIdle, OnInitialize, OnRuntimeUpgrade, OnTimestampSet,
+	Hooks, IntegrityTest, OnFinalize, OnGenesis, OnIdle, OnInitialize, OnRuntimeUpgrade,
+	OnTimestampSet,
 };
 #[cfg(feature = "try-runtime")]
 pub use hooks::{OnRuntimeUpgradeHelpersExt, ON_RUNTIME_UPGRADE_PREFIX};

frame/support/src/traits/filter.rs

@@ -180,17 +180,6 @@ macro_rules! impl_filter_stack {
 	}
 }
 
-/// Type that provide some integrity tests.
-///
-/// This implemented for modules by `decl_module`.
-#[impl_trait_for_tuples::impl_for_tuples(30)]
-pub trait IntegrityTest {
-	/// Run integrity test.
-	///
-	/// The test is not executed in a externalities provided environment.
-	fn integrity_test() {}
-}
-
 #[cfg(test)]
 pub mod test_impl_filter_stack {
 	use super::*;

frame/support/src/traits/hooks.rs

@@ -38,7 +38,9 @@ pub trait OnInitialize<BlockNumber> {
 	}
 }
 
-#[impl_for_tuples(30)]
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
 impl<BlockNumber: Clone> OnInitialize<BlockNumber> for Tuple {
 	fn on_initialize(n: BlockNumber) -> crate::weights::Weight {
 		let mut weight = 0;
@@ -50,7 +52,9 @@ impl<BlockNumber: Clone> OnInitialize<BlockNumber> for Tuple {
 /// The block finalization trait.
 ///
 /// Implementing this lets you express what should happen for your pallet when the block is ending.
-#[impl_for_tuples(30)]
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
 pub trait OnFinalize<BlockNumber> {
 	/// The block is being finalized. Implement to have something happen.
 	///
@@ -79,7 +83,9 @@ pub trait OnIdle<BlockNumber> {
 	}
 }
 
-#[impl_for_tuples(30)]
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
 impl<BlockNumber: Copy + AtLeast32BitUnsigned> OnIdle<BlockNumber> for Tuple {
 	fn on_idle(n: BlockNumber, remaining_weight: crate::weights::Weight) -> crate::weights::Weight {
 		let on_idle_functions: &[fn(
@@ -105,7 +111,9 @@ impl<BlockNumber: Copy + AtLeast32BitUnsigned> OnIdle<BlockNumber> for Tuple {
 /// Implementing this trait for a pallet let's you express operations that should
 /// happen at genesis. It will be called in an externalities provided environment and
 /// will see the genesis state after all pallets have written their genesis state.
-#[impl_for_tuples(30)]
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
 pub trait OnGenesis {
 	/// Something that should happen at genesis.
 	fn on_genesis() {}
@@ -187,7 +195,9 @@ pub trait OnRuntimeUpgrade {
 	}
 }
 
-#[impl_for_tuples(30)]
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
 impl OnRuntimeUpgrade for Tuple {
 	fn on_runtime_upgrade() -> crate::weights::Weight {
 		let mut weight = 0;
@@ -210,6 +220,19 @@ impl OnRuntimeUpgrade for Tuple {
 	}
 }
 
+/// Type that provide some integrity tests.
+///
+/// This implemented for modules by `decl_module`.
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
+pub trait IntegrityTest {
+	/// Run integrity test.
+	///
+	/// The test is not executed in a externalities provided environment.
+	fn integrity_test() {}
+}
+
 /// The pallet hooks trait. Implementing this lets you express some logic to execute.
 pub trait Hooks<BlockNumber> {
 	/// The block is being finalized. Implement to have something happen.
@@ -321,7 +344,9 @@ pub trait GenesisBuild<T, I = ()>: Default + sp_runtime::traits::MaybeSerializeD
 }
 
 /// A trait which is called when the timestamp is set in the runtime.
-#[impl_for_tuples(30)]
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
 pub trait OnTimestampSet<Moment> {
 	/// Called when the timestamp is set.
 	fn on_timestamp_set(moment: Moment);

frame/support/src/traits/members.rs

@@ -17,6 +17,7 @@
 
 //! Traits for dealing with the idea of membership.
 
+use impl_trait_for_tuples::impl_for_tuples;
 use sp_std::{marker::PhantomData, prelude::*};
 
 /// A trait for querying whether a type can be said to "contain" a value.
@@ -25,7 +26,9 @@ pub trait Contains<T> {
 	fn contains(t: &T) -> bool;
 }
 
-#[impl_trait_for_tuples::impl_for_tuples(1, 30)]
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
 impl<T> Contains<T> for Tuple {
 	fn contains(t: &T) -> bool {
 		for_tuples!( #(
@@ -41,7 +44,9 @@ pub trait ContainsPair<A, B> {
 	fn contains(a: &A, b: &B) -> bool;
 }
 
-#[impl_trait_for_tuples::impl_for_tuples(0, 30)]
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
 impl<A, B> ContainsPair<A, B> for Tuple {
 	fn contains(a: &A, b: &B) -> bool {
 		for_tuples!( #(

frame/support/src/traits/metadata.rs

@@ -18,6 +18,7 @@
 //! Traits for managing information attached to pallets and their constituents.
 
 use codec::{Decode, Encode};
+use impl_trait_for_tuples::impl_for_tuples;
 use sp_runtime::RuntimeDebug;
 use sp_std::prelude::*;
 
@@ -70,40 +71,22 @@ pub trait PalletsInfoAccess {
 	///
 	/// You probably don't want this function but `infos()` instead.
 	fn count() -> usize {
-		0
+		// for backwards compatibility with XCM-3, Mark is deprecated.
+		Self::infos().len()
 	}
 
-	/// Extend the given vector by all of the pallets' information that this type represents.
-	///
-	/// You probably don't want this function but `infos()` instead.
-	fn accumulate(_accumulator: &mut Vec<PalletInfoData>) {}
-
 	/// All of the pallets' information that this type represents.
-	fn infos() -> Vec<PalletInfoData> {
-		let mut result = Vec::with_capacity(Self::count());
-		Self::accumulate(&mut result);
-		result
-	}
+	fn infos() -> Vec<PalletInfoData>;
 }
 
-impl PalletsInfoAccess for () {}
-impl<T: PalletsInfoAccess> PalletsInfoAccess for (T,) {
-	fn count() -> usize {
-		T::count()
-	}
-	fn accumulate(acc: &mut Vec<PalletInfoData>) {
-		T::accumulate(acc)
-	}
-}
-
-impl<T1: PalletsInfoAccess, T2: PalletsInfoAccess> PalletsInfoAccess for (T1, T2) {
-	fn count() -> usize {
-		T1::count() + T2::count()
-	}
-	fn accumulate(acc: &mut Vec<PalletInfoData>) {
-		// The AllPallets type tuplises the pallets in reverse order, so we unreverse them here.
-		T2::accumulate(acc);
-		T1::accumulate(acc);
+#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))]
+#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))]
+#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))]
+impl PalletsInfoAccess for Tuple {
+	fn infos() -> Vec<PalletInfoData> {
+		let mut res = vec![];
+		for_tuples!( #( res.extend(Tuple::infos()); )* );
+		res
 	}
 }