From 3f83d6990b9f628917c46d88f7550c5885225ddc Mon Sep 17 00:00:00 2001 From: alindima Date: Thu, 25 Jan 2024 17:02:15 +0200 Subject: [PATCH 1/5] bump reed-solomon-novelpoly version also remove some dead code and deduplicate some error handling --- Cargo.lock | 37 ++---------- polkadot/erasure-coding/Cargo.toml | 3 +- polkadot/erasure-coding/src/lib.rs | 97 ++++++------------------------ 3 files changed, 25 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f685fcaa7021..0d63a32284ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14554,14 +14554,12 @@ dependencies = [ [[package]] name = "reed-solomon-novelpoly" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd8f48b2066e9f69ab192797d66da804d1935bf22763204ed3675740cb0f221" +version = "2.0.0" +source = "git+https://github.com/paritytech/reed-solomon-novelpoly.git?branch=alindima/release-2.0.0#ae76ce85a43f3abc5bf2676b5f1e20e5da7c9ba2" dependencies = [ "derive_more", "fs-err", - "itertools 0.10.5", - "static_init 0.5.2", + "static_init", "thiserror", ] @@ -16504,7 +16502,7 @@ dependencies = [ "sp-transaction-storage-proof", "sp-trie", "sp-version", - "static_init 1.0.3", + "static_init", "substrate-prometheus-endpoint", "substrate-test-runtime", "substrate-test-runtime-client", @@ -19285,18 +19283,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "static_init" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b73400442027c4adedda20a9f9b7945234a5bd8d5f7e86da22bd5d0622369c" -dependencies = [ - "cfg_aliases", - "libc", - "parking_lot 0.11.2", - "static_init_macro 0.5.0", -] - [[package]] name = "static_init" version = "1.0.3" @@ -19308,23 +19294,10 @@ dependencies = [ "libc", "parking_lot 0.11.2", "parking_lot_core 0.8.6", - "static_init_macro 1.0.2", + "static_init_macro", "winapi", ] -[[package]] -name = "static_init_macro" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2261c91034a1edc3fc4d1b80e89d82714faede0515c14a75da10cb941546bbf" -dependencies = [ - "cfg_aliases", - "memchr", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "static_init_macro" version = "1.0.2" diff --git a/polkadot/erasure-coding/Cargo.toml b/polkadot/erasure-coding/Cargo.toml index f174f8ad0cf4..55d49e678aeb 100644 --- a/polkadot/erasure-coding/Cargo.toml +++ b/polkadot/erasure-coding/Cargo.toml @@ -12,7 +12,8 @@ workspace = true [dependencies] polkadot-primitives = { path = "../primitives" } polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../node/primitives" } -novelpoly = { package = "reed-solomon-novelpoly", version = "1.0.0" } +# novelpoly = { package = "reed-solomon-novelpoly", version = "1.0.0" } +novelpoly = { package = "reed-solomon-novelpoly", git = "https://github.com/paritytech/reed-solomon-novelpoly.git", branch = "alindima/release-2.0.0" } parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive", "std"] } sp-core = { path = "../../substrate/primitives/core" } sp-trie = { path = "../../substrate/primitives/trie" } diff --git a/polkadot/erasure-coding/src/lib.rs b/polkadot/erasure-coding/src/lib.rs index 36847b463715..e5155df4beba 100644 --- a/polkadot/erasure-coding/src/lib.rs +++ b/polkadot/erasure-coding/src/lib.rs @@ -83,6 +83,20 @@ pub enum Error { UnknownCodeParam, } +impl From for Error { + fn from(error: novelpoly::Error) -> Self { + match error { + novelpoly::Error::NeedMoreShards { .. } => Self::NotEnoughChunks, + novelpoly::Error::ParamterMustBePowerOf2 { .. } => Self::UnevenLength, + novelpoly::Error::WantedShardCountTooHigh(_) => Self::TooManyValidators, + novelpoly::Error::WantedShardCountTooLow(_) => Self::NotEnoughValidators, + novelpoly::Error::PayloadSizeIsZero { .. } => Self::BadPayload, + novelpoly::Error::InconsistentShardLengths { .. } => Self::NonUniformChunks, + _ => Self::UnknownReconstruction, + } + } +} + /// Obtain a threshold of chunks that should be enough to recover the data. pub const fn recovery_threshold(n_validators: usize) -> Result { if n_validators > MAX_VALIDATORS { @@ -166,42 +180,17 @@ where { let params = code_params(n_validators)?; let mut received_shards: Vec> = vec![None; n_validators]; - let mut shard_len = None; for (chunk_data, chunk_idx) in chunks.into_iter().take(n_validators) { - if chunk_idx >= n_validators { - return Err(Error::ChunkIndexOutOfBounds { chunk_index: chunk_idx, n_validators }) - } - - let shard_len = shard_len.get_or_insert_with(|| chunk_data.len()); - - if *shard_len % 2 != 0 { + if chunk_data.len() % 2 != 0 { return Err(Error::UnevenLength) } - if *shard_len != chunk_data.len() || *shard_len == 0 { - return Err(Error::NonUniformChunks) - } - received_shards[chunk_idx] = Some(WrappedShard::new(chunk_data.to_vec())); } - let res = params.make_encoder().reconstruct(received_shards); - - let payload_bytes = match res { - Err(e) => match e { - novelpoly::Error::NeedMoreShards { .. } => return Err(Error::NotEnoughChunks), - novelpoly::Error::ParamterMustBePowerOf2 { .. } => return Err(Error::UnevenLength), - novelpoly::Error::WantedShardCountTooHigh(_) => return Err(Error::TooManyValidators), - novelpoly::Error::WantedShardCountTooLow(_) => return Err(Error::NotEnoughValidators), - novelpoly::Error::PayloadSizeIsZero { .. } => return Err(Error::BadPayload), - novelpoly::Error::InconsistentShardLengths { .. } => - return Err(Error::NonUniformChunks), - _ => return Err(Error::UnknownReconstruction), - }, - Ok(payload_bytes) => payload_bytes, - }; - - Decode::decode(&mut &payload_bytes[..]).or_else(|_e| Err(Error::BadPayload)) + let payload_bytes = params.make_encoder().reconstruct(received_shards)?; + + Decode::decode(&mut &payload_bytes[..]).map_err(|_| Error::BadPayload) } /// An iterator that yields merkle branches and chunk data for all chunks to @@ -294,56 +283,6 @@ pub fn branch_hash(root: &H256, branch_nodes: &Proof, index: usize) -> Result

{ - remaining_len: usize, - shards: I, - cur_shard: Option<(&'a [u8], usize)>, -} - -impl<'a, I: Iterator> parity_scale_codec::Input for ShardInput<'a, I> { - fn remaining_len(&mut self) -> Result, parity_scale_codec::Error> { - Ok(Some(self.remaining_len)) - } - - fn read(&mut self, into: &mut [u8]) -> Result<(), parity_scale_codec::Error> { - let mut read_bytes = 0; - - loop { - if read_bytes == into.len() { - break - } - - let cur_shard = self.cur_shard.take().or_else(|| self.shards.next().map(|s| (s, 0))); - let (active_shard, mut in_shard) = match cur_shard { - Some((s, i)) => (s, i), - None => break, - }; - - if in_shard >= active_shard.len() { - continue - } - - let remaining_len_out = into.len() - read_bytes; - let remaining_len_shard = active_shard.len() - in_shard; - - let write_len = std::cmp::min(remaining_len_out, remaining_len_shard); - into[read_bytes..][..write_len].copy_from_slice(&active_shard[in_shard..][..write_len]); - - in_shard += write_len; - read_bytes += write_len; - self.cur_shard = Some((active_shard, in_shard)) - } - - self.remaining_len -= read_bytes; - if read_bytes == into.len() { - Ok(()) - } else { - Err("slice provided too big for input".into()) - } - } -} - #[cfg(test)] mod tests { use super::*; From d588d043ea85bb59a5657aa9bc3c1e8e7dc43fec Mon Sep 17 00:00:00 2001 From: alindima Date: Thu, 25 Jan 2024 17:30:44 +0200 Subject: [PATCH 2/5] switch to released version --- Cargo.lock | 3 ++- polkadot/erasure-coding/Cargo.toml | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d63a32284ab..057a1cc59adf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14555,7 +14555,8 @@ dependencies = [ [[package]] name = "reed-solomon-novelpoly" version = "2.0.0" -source = "git+https://github.com/paritytech/reed-solomon-novelpoly.git?branch=alindima/release-2.0.0#ae76ce85a43f3abc5bf2676b5f1e20e5da7c9ba2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87413ebb313323d431e85d0afc5a68222aaed972843537cbfe5f061cf1b4bcab" dependencies = [ "derive_more", "fs-err", diff --git a/polkadot/erasure-coding/Cargo.toml b/polkadot/erasure-coding/Cargo.toml index 55d49e678aeb..e0b5402ca486 100644 --- a/polkadot/erasure-coding/Cargo.toml +++ b/polkadot/erasure-coding/Cargo.toml @@ -12,8 +12,7 @@ workspace = true [dependencies] polkadot-primitives = { path = "../primitives" } polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../node/primitives" } -# novelpoly = { package = "reed-solomon-novelpoly", version = "1.0.0" } -novelpoly = { package = "reed-solomon-novelpoly", git = "https://github.com/paritytech/reed-solomon-novelpoly.git", branch = "alindima/release-2.0.0" } +novelpoly = { package = "reed-solomon-novelpoly", version = "2.0.0" } parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive", "std"] } sp-core = { path = "../../substrate/primitives/core" } sp-trie = { path = "../../substrate/primitives/trie" } From e142c792bca958542b844035ea10f8cb1c71808f Mon Sep 17 00:00:00 2001 From: alindima Date: Thu, 25 Jan 2024 17:36:26 +0200 Subject: [PATCH 3/5] add prdoc --- prdoc/pr_3065.prdoc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 prdoc/pr_3065.prdoc diff --git a/prdoc/pr_3065.prdoc b/prdoc/pr_3065.prdoc new file mode 100644 index 000000000000..fbd60f121a13 --- /dev/null +++ b/prdoc/pr_3065.prdoc @@ -0,0 +1,14 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Bump reed-solomon-novelpoly to version 2.0.0 + +author: alindima + +doc: + - audience: Node Dev + description: | + Use the new reed-solomon-novelpoly release. Brings some performance improvements. + +crates: + - name = "erasure-coding" From 5b770ff266103a95e470d35e080012ba3ab1170a Mon Sep 17 00:00:00 2001 From: alindima Date: Thu, 25 Jan 2024 17:40:51 +0200 Subject: [PATCH 4/5] fix prdoc --- prdoc/pr_3065.prdoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prdoc/pr_3065.prdoc b/prdoc/pr_3065.prdoc index fbd60f121a13..4c0bb739b06c 100644 --- a/prdoc/pr_3065.prdoc +++ b/prdoc/pr_3065.prdoc @@ -11,4 +11,4 @@ doc: Use the new reed-solomon-novelpoly release. Brings some performance improvements. crates: - - name = "erasure-coding" + - name: "erasure-coding" From dfecf2ab67c8987c8682f07f05a79b36335f7fb9 Mon Sep 17 00:00:00 2001 From: alindima Date: Thu, 25 Jan 2024 18:13:24 +0200 Subject: [PATCH 5/5] remove prdoc --- prdoc/pr_3065.prdoc | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 prdoc/pr_3065.prdoc diff --git a/prdoc/pr_3065.prdoc b/prdoc/pr_3065.prdoc deleted file mode 100644 index 4c0bb739b06c..000000000000 --- a/prdoc/pr_3065.prdoc +++ /dev/null @@ -1,14 +0,0 @@ -# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 -# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json - -title: Bump reed-solomon-novelpoly to version 2.0.0 - -author: alindima - -doc: - - audience: Node Dev - description: | - Use the new reed-solomon-novelpoly release. Brings some performance improvements. - -crates: - - name: "erasure-coding"