diff --git a/Cargo.lock b/Cargo.lock index 7bd0f3bf3d56..0930565d7cb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,6 +152,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf688625d06217d5b1bb0ea9d9c44a1635fd0ee3534466388d18203174f4d11" +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "ansi_term" version = "0.12.1" @@ -756,6 +762,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.73" @@ -844,6 +856,33 @@ dependencies = [ "winapi", ] +[[package]] +name = "ciborium" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" + +[[package]] +name = "ciborium-ll" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cid" version = "0.8.6" @@ -895,6 +934,18 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "3.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +dependencies = [ + "bitflags", + "clap_lex 0.2.4", + "indexmap", + "textwrap", +] + [[package]] name = "clap" version = "4.0.15" @@ -904,7 +955,7 @@ dependencies = [ "atty", "bitflags", "clap_derive", - "clap_lex", + "clap_lex 0.3.0", "once_cell", "strsim", "termcolor", @@ -923,6 +974,15 @@ dependencies = [ "syn", ] +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clap_lex" version = "0.3.0" @@ -1186,6 +1246,40 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap 3.2.23", + "criterion-plot", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-channel" version = "0.5.5" @@ -2226,7 +2320,7 @@ dependencies = [ "Inflector", "array-bytes", "chrono", - "clap", + "clap 4.0.15", "comfy-table", "frame-benchmarking", "frame-support", @@ -2817,6 +2911,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "handlebars" version = "4.2.2" @@ -4808,6 +4908,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.2.3" @@ -6313,7 +6419,7 @@ dependencies = [ name = "polkadot-cli" version = "0.9.33" dependencies = [ - "clap", + "clap 4.0.15", "frame-benchmarking-cli", "futures", "log", @@ -6456,6 +6562,7 @@ dependencies = [ name = "polkadot-erasure-coding" version = "0.9.33" dependencies = [ + "criterion", "parity-scale-codec", "polkadot-node-primitives", "polkadot-primitives", @@ -7570,7 +7677,7 @@ version = "0.9.33" dependencies = [ "assert_matches", "async-trait", - "clap", + "clap 4.0.15", "color-eyre", "futures", "futures-timer", @@ -7716,7 +7823,7 @@ dependencies = [ name = "polkadot-voter-bags" version = "0.9.33" dependencies = [ - "clap", + "clap 4.0.15", "generate-bags", "kusama-runtime", "polkadot-runtime", @@ -8343,7 +8450,7 @@ checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" name = "remote-ext-tests-bags-list" version = "0.9.33" dependencies = [ - "clap", + "clap 4.0.15", "frame-system", "kusama-runtime", "kusama-runtime-constants", @@ -8844,7 +8951,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#934d42aefb51 dependencies = [ "array-bytes", "chrono", - "clap", + "clap 4.0.15", "fdlimit", "futures", "libp2p", @@ -10972,7 +11079,7 @@ name = "staking-miner" version = "0.9.33" dependencies = [ "assert_cmd", - "clap", + "clap 4.0.15", "exitcode", "frame-election-provider-support", "frame-remote-externalities", @@ -11432,7 +11539,7 @@ dependencies = [ name = "test-parachain-adder-collator" version = "0.9.33" dependencies = [ - "clap", + "clap 4.0.15", "futures", "futures-timer", "log", @@ -11479,7 +11586,7 @@ dependencies = [ name = "test-parachain-undying-collator" version = "0.9.33" dependencies = [ - "clap", + "clap 4.0.15", "futures", "futures-timer", "log", @@ -11525,6 +11632,12 @@ dependencies = [ "sp-weights", ] +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + [[package]] name = "thiserror" version = "1.0.37" @@ -12066,7 +12179,7 @@ name = "try-runtime-cli" version = "0.10.0-dev" source = "git+https://github.com/paritytech/substrate?branch=master#934d42aefb51b797ee9ef41270bc041b1c1c6025" dependencies = [ - "clap", + "clap 4.0.15", "frame-remote-externalities", "frame-try-runtime", "hex", diff --git a/erasure-coding/Cargo.toml b/erasure-coding/Cargo.toml index 6d59b6a4b01d..868d1197efd7 100644 --- a/erasure-coding/Cargo.toml +++ b/erasure-coding/Cargo.toml @@ -12,3 +12,10 @@ parity-scale-codec = { version = "3.1.5", default-features = false, features = [ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" } thiserror = "1.0.31" + +[dev-dependencies] +criterion = { version = "0.4.0", default-features = false, features = ["cargo_bench_support"] } + +[[bench]] +name = "scaling_with_validators" +harness = false diff --git a/erasure-coding/benches/README.md b/erasure-coding/benches/README.md new file mode 100644 index 000000000000..e643643229e8 --- /dev/null +++ b/erasure-coding/benches/README.md @@ -0,0 +1,39 @@ +### Run benches +``` +$ cd erasure-coding # ensure you are in the right directory +$ cargo bench +``` + +### `scaling_with_validators` + +This benchmark evaluates the performance of constructing the chunks and the erasure root from PoV and +reconstructing the PoV from chunks. You can see the results of running this bench on 5950x below. +Interestingly, with `10_000` chunks (validators) its slower than with `50_000` for both construction +and reconstruction. +``` +construct/200 time: [93.924 ms 94.525 ms 95.214 ms] + thrpt: [52.513 MiB/s 52.896 MiB/s 53.234 MiB/s] +construct/500 time: [111.25 ms 111.52 ms 111.80 ms] + thrpt: [44.721 MiB/s 44.837 MiB/s 44.946 MiB/s] +construct/1000 time: [117.37 ms 118.28 ms 119.21 ms] + thrpt: [41.941 MiB/s 42.273 MiB/s 42.601 MiB/s] +construct/2000 time: [125.05 ms 125.72 ms 126.38 ms] + thrpt: [39.564 MiB/s 39.772 MiB/s 39.983 MiB/s] +construct/10000 time: [270.46 ms 275.11 ms 279.81 ms] + thrpt: [17.869 MiB/s 18.174 MiB/s 18.487 MiB/s] +construct/50000 time: [205.86 ms 209.66 ms 213.64 ms] + thrpt: [23.404 MiB/s 23.848 MiB/s 24.288 MiB/s] + +reconstruct/200 time: [180.73 ms 184.09 ms 187.73 ms] + thrpt: [26.634 MiB/s 27.160 MiB/s 27.666 MiB/s] +reconstruct/500 time: [195.59 ms 198.58 ms 201.76 ms] + thrpt: [24.781 MiB/s 25.179 MiB/s 25.564 MiB/s] +reconstruct/1000 time: [207.92 ms 211.57 ms 215.57 ms] + thrpt: [23.195 MiB/s 23.633 MiB/s 24.048 MiB/s] +reconstruct/2000 time: [218.59 ms 223.68 ms 229.18 ms] + thrpt: [21.817 MiB/s 22.354 MiB/s 22.874 MiB/s] +reconstruct/10000 time: [496.35 ms 505.17 ms 515.42 ms] + thrpt: [9.7008 MiB/s 9.8977 MiB/s 10.074 MiB/s] +reconstruct/50000 time: [276.56 ms 277.53 ms 278.58 ms] + thrpt: [17.948 MiB/s 18.016 MiB/s 18.079 MiB/s] +``` diff --git a/erasure-coding/benches/scaling_with_validators.rs b/erasure-coding/benches/scaling_with_validators.rs new file mode 100644 index 000000000000..d361c23537c2 --- /dev/null +++ b/erasure-coding/benches/scaling_with_validators.rs @@ -0,0 +1,74 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use polkadot_primitives::v2::Hash; +use std::time::Duration; + +fn chunks(n_validators: usize, pov: &Vec) -> Vec> { + polkadot_erasure_coding::obtain_chunks(n_validators, pov).unwrap() +} + +fn erasure_root(n_validators: usize, pov: &Vec) -> Hash { + let chunks = chunks(n_validators, pov); + polkadot_erasure_coding::branches(&chunks).root() +} + +fn construct_and_reconstruct_5mb_pov(c: &mut Criterion) { + const N_VALIDATORS: [usize; 6] = [200, 500, 1000, 2000, 10_000, 50_000]; + + const KB: usize = 1024; + const MB: usize = 1024 * KB; + + let pov = vec![0xfe; 5 * MB]; + + let mut group = c.benchmark_group("construct"); + for n_validators in N_VALIDATORS { + let expected_root = erasure_root(n_validators, &pov); + + group.throughput(Throughput::Bytes(pov.len() as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(n_validators), + &n_validators, + |b, &n| { + b.iter(|| { + let root = erasure_root(n, &pov); + assert_eq!(root, expected_root); + }); + }, + ); + } + group.finish(); + + let mut group = c.benchmark_group("reconstruct"); + for n_validators in N_VALIDATORS { + let all_chunks = chunks(n_validators, &pov); + + let mut c: Vec<_> = all_chunks.iter().enumerate().map(|(i, c)| (&c[..], i)).collect(); + let last_chunks = c.split_off((c.len() - 1) * 2 / 3); + + group.throughput(Throughput::Bytes(pov.len() as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(n_validators), + &n_validators, + |b, &n| { + b.iter(|| { + let _pov: Vec = + polkadot_erasure_coding::reconstruct(n, last_chunks.clone()).unwrap(); + }); + }, + ); + } + group.finish(); +} + +fn criterion_config() -> Criterion { + Criterion::default() + .sample_size(15) + .warm_up_time(Duration::from_millis(200)) + .measurement_time(Duration::from_secs(3)) +} + +criterion_group!( + name = re_construct; + config = criterion_config(); + targets = construct_and_reconstruct_5mb_pov, +); +criterion_main!(re_construct);