From 7777f7a1db946fb59d96267b88b619e7f09eef4f Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 6 Sep 2024 11:49:15 +0800 Subject: [PATCH 01/79] init in_memory_tree --- Cargo.toml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f4dc52f..de8bcdc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,4 +10,11 @@ kvdb-rocksdb = "0.19" kvdb = "0.13" parking_lot = "0.12" -ethereum-types = "0.12" \ No newline at end of file +ethereum-types = "0.12" + +in-memory-tree = { path = "src/in_memory_tree" } + +[workspace] +members = [ + "src/in_memory_tree" +] \ No newline at end of file From 95f417759fa6c09ea33463ebfa283b731f9f2a14 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:01:42 +0800 Subject: [PATCH 02/79] replace pending_part with VersionedHashMap --- src/in_memory_tree/Cargo.toml | 12 + src/in_memory_tree/src/commit_tree.rs | 239 ++++++++++++++ src/in_memory_tree/src/error.rs | 13 + src/in_memory_tree/src/lib.rs | 6 + src/in_memory_tree/src/versioned_hash_map.rs | 300 ++++++++++++++++++ .../versioned_flat_key_value/mod.rs | 11 +- 6 files changed, 577 insertions(+), 4 deletions(-) create mode 100644 src/in_memory_tree/Cargo.toml create mode 100644 src/in_memory_tree/src/commit_tree.rs create mode 100644 src/in_memory_tree/src/error.rs create mode 100644 src/in_memory_tree/src/lib.rs create mode 100644 src/in_memory_tree/src/versioned_hash_map.rs diff --git a/src/in_memory_tree/Cargo.toml b/src/in_memory_tree/Cargo.toml new file mode 100644 index 0000000..05d4337 --- /dev/null +++ b/src/in_memory_tree/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "in-memory-tree" +version = "0.1.0" +edition = "2021" + +[dependencies] +slab = "0.4.9" +thiserror = "1" + +[dev-dependencies] +rand = "0.8.0" +rand_distr = "0.4.0" \ No newline at end of file diff --git a/src/in_memory_tree/src/commit_tree.rs b/src/in_memory_tree/src/commit_tree.rs new file mode 100644 index 0000000..790f049 --- /dev/null +++ b/src/in_memory_tree/src/commit_tree.rs @@ -0,0 +1,239 @@ +use std::collections::{BTreeSet, HashMap}; +use std::{fmt::Debug, hash::Hash}; + +use slab::Slab; + +use crate::TreeError; + +type SlabIndex = usize; + +pub struct TreeNode { + parent: Option, + children: BTreeSet, + + height: usize, + + commit_id: CommitId, + // before current node, the old value of this key is modified by which commit_id, + // if none, this key is absent before current node + // here must use CommitID instead of SlabIndex (which may be reused, see slab doc) + modifications: Vec<(Key, Option, Option)>, +} + +pub struct Tree { + nodes: Slab>, + index_map: HashMap, +} + +impl + Tree +{ + pub fn new() -> Self { + Tree { + nodes: Slab::new(), + index_map: HashMap::new(), + } + } + + fn commit_id_to_slab_index( + &self, + commit_id: CommitId, + ) -> Result> { + let slab_index = *self + .index_map + .get(&commit_id) + .ok_or_else(|| TreeError::CommitIDNotFound(commit_id))?; + Ok(slab_index) + } + + fn slab_index_to_node(&self, slab_index: SlabIndex) -> &TreeNode { + &self.nodes[slab_index] + } + + fn has_root(&self) -> bool { + !self.index_map.is_empty() + } + + pub fn get_parent_commit_id(&self, node: &TreeNode) -> Option { + node.parent + .and_then(|p_slab_index| Some(self.nodes[p_slab_index].commit_id)) + } +} + +impl + Tree +{ + pub fn add_node( + &mut self, + commit_id: CommitId, + parent_commit_id: Option, + modifications: Vec<(Key, Option, Option)>, + ) -> Result<(), TreeError> { + // return error if Some(parent_commit_id) but parent_commit_id does not exist + let (parent_slab_index, parent_height) = if let Some(parent_commit_id) = parent_commit_id { + let p_slab_index = self.commit_id_to_slab_index(parent_commit_id)?; + let p_height = self.slab_index_to_node(p_slab_index).height; + (Some(p_slab_index), p_height) + } else { + // return error if want to add root but there has been a root + if self.has_root() { + return Err(TreeError::MultipleRootsNotAllowed); + } + (None, 0) + }; + // return error if commit_id exists + if self.index_map.contains_key(&commit_id) { + return Err(TreeError::CommitIdAlreadyExists(commit_id)); + } + let node = TreeNode::new(commit_id, parent_slab_index, parent_height, modifications); + + let slab_index = self.nodes.insert(node); + self.index_map.insert(commit_id, slab_index); + if let Some(parent_slab_index) = parent_slab_index { + self.nodes[parent_slab_index].children.insert(slab_index); + } + Ok(()) + } + + fn bfs_subtree(&mut self, subroot_slab_index: SlabIndex) -> Vec { + let mut slab_indices = vec![subroot_slab_index]; + let mut head = 0; + while head < slab_indices.len() { + let node = self.slab_index_to_node(slab_indices[head]); + + for &child_index in &node.children { + slab_indices.push(child_index); + } + + head += 1; + } + + slab_indices + } + + pub fn remove_subtree( + &mut self, + subroot_commit_id: CommitId, + ) -> Result<(), TreeError> { + let subroot_slab_index = self.commit_id_to_slab_index(subroot_commit_id)?; + + let to_remove = self.bfs_subtree(subroot_slab_index); + + // only subroot should update its parent, + // since, for other nodes to be removed, their parents will also be removed + if let Some(parent_slab_index) = self.slab_index_to_node(subroot_slab_index).parent { + let parent = &mut self.nodes[parent_slab_index]; + parent.children.remove(&subroot_slab_index); + } + + for &index in to_remove.iter() { + let node = self.nodes.remove(index); + self.index_map.remove(&node.commit_id); + } + + Ok(()) + } +} + +impl + Tree +{ + pub fn find_path( + &self, + target_commit_id: CommitId, + ) -> Result)>, TreeError> { + let target_slab_index = self.commit_id_to_slab_index(target_commit_id)?; + let mut target_node = self.slab_index_to_node(target_slab_index); + let mut commits_rev = HashMap::new(); + loop { + target_node.target_up(&mut commits_rev); + if target_node.parent.is_none() { + break; + } + target_node = self.slab_index_to_node(target_node.parent.unwrap()); + } + Ok(commits_rev) + } + + // correctness based on single root + pub fn lca( + &self, + current_commit_id: CommitId, + target_commit_id: CommitId, + ) -> Result< + ( + HashMap>, + HashMap)>, + ), + TreeError, + > { + let current_slab_index = self.commit_id_to_slab_index(current_commit_id)?; + let target_slab_index = self.commit_id_to_slab_index(target_commit_id)?; + let mut current_node = self.slab_index_to_node(current_slab_index); + let mut target_node = self.slab_index_to_node(target_slab_index); + let mut rollbacks = HashMap::new(); + let mut commits_rev = HashMap::new(); + while current_node.height > target_node.height { + current_node.current_up(&mut rollbacks); + current_node = self.slab_index_to_node(current_node.parent.unwrap()); + } + while target_node.height > current_node.height { + target_node.target_up(&mut commits_rev); + target_node = self.slab_index_to_node(target_node.parent.unwrap()); + } + while current_node.commit_id != target_node.commit_id { + current_node.current_up(&mut rollbacks); + current_node = self.slab_index_to_node(current_node.parent.unwrap()); + target_node.target_up(&mut commits_rev); + target_node = self.slab_index_to_node(target_node.parent.unwrap()); + } + // rollbacks or commits_rev may be empty, + // they contain current and target (if they are not lca), respectively, + // but they do not contain lca + Ok((rollbacks, commits_rev)) + } +} + +impl + TreeNode +{ + pub fn new( + commit_id: CommitId, + parent: Option, + parent_height: usize, + modifications: Vec<(Key, Option, Option)>, + ) -> Self { + Self { + height: parent_height + 1, + commit_id, + parent, + children: BTreeSet::new(), + modifications, + } + } + + pub fn get_commit_id(&self) -> CommitId { + self.commit_id + } + + pub fn get_modifications( + &self, + ) -> impl Iterator, Option)> { + self.modifications.iter() + } + + pub fn current_up(&self, rollbacks: &mut HashMap>) { + for (key, _, old_commit_id) in self.get_modifications() { + rollbacks.insert(key.clone(), *old_commit_id); + } + } + + pub fn target_up(&self, commits_rev: &mut HashMap)>) { + let commit_id = self.commit_id; + for (key, value, _) in self.get_modifications() { + commits_rev + .entry(key.clone()) + .or_insert_with(|| (commit_id, value.clone())); + } + } +} diff --git a/src/in_memory_tree/src/error.rs b/src/in_memory_tree/src/error.rs new file mode 100644 index 0000000..695ab6b --- /dev/null +++ b/src/in_memory_tree/src/error.rs @@ -0,0 +1,13 @@ +use std::{fmt::Debug, hash::Hash}; + +use thiserror::Error; + +#[derive(Debug, PartialEq, Error)] +pub enum TreeError { + #[error("commit id not found")] + CommitIDNotFound(CommitId), + #[error("multiple roots are not allowed")] + MultipleRootsNotAllowed, + #[error("commit id already exists")] + CommitIdAlreadyExists(CommitId), +} diff --git a/src/in_memory_tree/src/lib.rs b/src/in_memory_tree/src/lib.rs new file mode 100644 index 0000000..2cd91af --- /dev/null +++ b/src/in_memory_tree/src/lib.rs @@ -0,0 +1,6 @@ +pub mod commit_tree; +pub mod error; +pub mod versioned_hash_map; + +pub use error::TreeError; +pub use versioned_hash_map::VersionedHashMap; diff --git a/src/in_memory_tree/src/versioned_hash_map.rs b/src/in_memory_tree/src/versioned_hash_map.rs new file mode 100644 index 0000000..8217faf --- /dev/null +++ b/src/in_memory_tree/src/versioned_hash_map.rs @@ -0,0 +1,300 @@ +use std::{ + collections::{BTreeMap, HashMap}, + fmt::Debug, + hash::Hash, +}; + +use crate::{commit_tree::Tree, TreeError}; + +pub struct VersionedHashMap< + Key: Eq + Hash + Clone + Ord, + CommitId: Debug + Eq + Hash + Copy, + Value: Clone, +> { + history: HashMap<(Key, CommitId), Option>, + tree: Tree, + + current: BTreeMap)>, + current_node: Option, +} + +impl + VersionedHashMap +{ + pub fn new() -> Self { + VersionedHashMap { + current: BTreeMap::new(), + history: HashMap::new(), + tree: Tree::new(), + current_node: None, + } + } +} + +impl + VersionedHashMap +{ + pub fn add_node( + &mut self, + updates: Vec<(Key, Option)>, + commit_id: CommitId, + parent_commit_id: Option, + ) -> Result<(), TreeError> { + // let parent to be self.current + self.walk_to_node(parent_commit_id)?; + assert_eq!(parent_commit_id, self.current_node); + // add node + let mut modifications = Vec::new(); + for (key, value) in updates.into_iter() { + self.history.insert((key.clone(), commit_id), value.clone()); + let old_commit_id = { + if let Some((old_commit_id, _)) = + self.current.insert(key.clone(), (commit_id, value.clone())) + { + Some(old_commit_id) + } else { + None + } + }; + modifications.push((key, value, old_commit_id)); + } + self.tree + .add_node(commit_id, parent_commit_id, modifications)?; + Ok(()) + } + + pub fn query( + &mut self, + commit_id: CommitId, + key: &Key, + ) -> Result, TreeError> { + // let queried node to be self.current + self.walk_to_node_unchecked(commit_id)?; + assert_eq!(Some(commit_id), self.current_node); + // query + let value = self + .current + .get(key) + .and_then(|(_, value)| Some(value.clone())); + if let Some(Some(value_inner)) = value { + Ok(Some(value_inner)) + } else { + Ok(None) + } + } + + pub fn query_range<'a>( + &'a mut self, + commit_id: CommitId, + key: Key, + ) -> Result + 'a>, TreeError> { + // let queried node to be self.current + self.walk_to_node_unchecked(commit_id)?; + assert_eq!(Some(commit_id), self.current_node); + // query + let range = self + .current + .range(key..) + .filter_map(|(key, (_, opt_value))| { + if let Some(value) = opt_value { + Some((key, value)) + } else { + None + } + }); + Ok(Box::new(range)) + } +} + +impl + VersionedHashMap +{ + fn walk_to_node( + &mut self, + target_commit_id: Option, + ) -> Result<(), TreeError> { + if target_commit_id.is_none() { + self.current = BTreeMap::new(); + self.current_node = None; + } else { + self.walk_to_node_unchecked(target_commit_id.unwrap())?; + } + Ok(()) + } + fn walk_to_node_unchecked( + &mut self, + target_commit_id: CommitId, + ) -> Result<(), TreeError> { + let (rollbacks, commits_rev) = if self.current_node.is_none() { + let commits_rev = self.tree.find_path(target_commit_id)?; + (HashMap::new(), commits_rev) + } else { + self.tree + .lca(self.current_node.unwrap(), target_commit_id)? + }; + self.rollback_without_node_update(rollbacks); + self.commit_without_node_update(commits_rev); + self.current_node = Some(target_commit_id); + Ok(()) + } + + fn commit_without_node_update(&mut self, commits_rev: HashMap)>) { + for (key, (commit_id, value)) in commits_rev.into_iter() { + self.current.insert(key, (commit_id, value)); + } + } + + fn rollback_without_node_update(&mut self, rollbacks: HashMap>) { + for (key, old_commit_id) in rollbacks.into_iter() { + match old_commit_id { + None => { + self.current.remove(&key); + } + Some(old_commit_id) => { + if let Some(value) = self.history.get(&(key.clone(), old_commit_id)) { + self.current + .insert(key.clone(), (old_commit_id, value.clone())); + } else { + unreachable!( + "A modification recorded in a rollbacked commit is absent in history" + ); + } + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::{rngs::ThreadRng, Rng}; + use rand_distr::{Distribution, Uniform}; + + pub type Key = Vec; + pub type Value = Vec; + pub type CommitId = u64; + + fn random_key_value(rng: &mut ThreadRng) -> (Key, Option) { + // use a small key range to achieve more key conflicts + let key_range = Uniform::from(0..10); + let key: Key = vec![key_range.sample(rng)]; + + let value: Option = if rng.gen_range(0..2) == 0 { + Some(vec![rng.gen::()]) + } else { + None + }; + + (key, value) + } + + fn generate_random_tree( + num_nodes: usize, + rng: &mut ThreadRng, + ) -> ( + Tree, + VersionedHashMap, + ) { + let mut forward_only_tree = Tree::new(); + let mut versioned_hash_map = VersionedHashMap::new(); + + for i in 1..=num_nodes as CommitId { + let parent_commit_id = if i == 1 { + None + } else { + Some(rng.gen_range(1..i)) + }; + let mut updates = Vec::new(); + for _ in 0..5 { + updates.push(random_key_value(rng)); + } + let updates_none = updates + .iter() + .map(|(key, value)| (key.clone(), value.clone(), None)) + .collect(); + + forward_only_tree + .add_node(i, parent_commit_id, updates_none) + .unwrap(); + versioned_hash_map + .add_node(updates, i, parent_commit_id) + .unwrap(); + } + (forward_only_tree, versioned_hash_map) + } + + #[test] + fn test_query() { + let num_nodes = 10; + let mut rng = rand::thread_rng(); + let (forward_only_tree, mut versioned_hash_map) = generate_random_tree(num_nodes, &mut rng); + for _ in 0..100 { + let commit_id = rng.gen_range(1..=num_nodes) as CommitId; + for ikey in 0..10 { + let key: Key = ikey.to_string().into_bytes(); + let result = versioned_hash_map.query(commit_id, &key).unwrap(); + let current = forward_only_tree.find_path(commit_id).unwrap(); + let answer = current.get(&key).and_then(|(_, value)| { + if value.is_some() { + Some(value.clone().unwrap()) + } else { + None + } + }); + assert_eq!(result, answer); + } + } + } + + #[test] + fn test_multiple_roots_err() { + let mut forward_only_tree = Tree::::new(); + let mut versioned_hash_map = VersionedHashMap::::new(); + + forward_only_tree.add_node(0, None, Vec::new()).unwrap(); + versioned_hash_map.add_node(Vec::new(), 0, None).unwrap(); + + assert_eq!( + forward_only_tree.add_node(1, None, Vec::new()), + Err(TreeError::MultipleRootsNotAllowed) + ); + assert_eq!( + versioned_hash_map.add_node(Vec::new(), 1, None), + Err(TreeError::MultipleRootsNotAllowed) + ); + } + + #[test] + fn test_commit_id_not_found_err() { + let mut forward_only_tree = Tree::::new(); + let mut versioned_hash_map = VersionedHashMap::::new(); + + assert_eq!( + forward_only_tree.add_node(1, Some(0), Vec::new()), + Err(TreeError::CommitIDNotFound(0)) + ); + assert_eq!( + versioned_hash_map.add_node(Vec::new(), 1, Some(0)), + Err(TreeError::CommitIDNotFound(0)) + ); + } + + #[test] + fn test_commit_id_already_exists_err() { + let mut forward_only_tree = Tree::::new(); + let mut versioned_hash_map = VersionedHashMap::::new(); + + forward_only_tree.add_node(0, None, Vec::new()).unwrap(); + versioned_hash_map.add_node(Vec::new(), 0, None).unwrap(); + + assert_eq!( + forward_only_tree.add_node(0, Some(0), Vec::new()), + Err(TreeError::CommitIdAlreadyExists(0)) + ); + assert_eq!( + versioned_hash_map.add_node(Vec::new(), 0, Some(0)), + Err(TreeError::CommitIdAlreadyExists(0)) + ); + } +} diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 45f84cf..6420b93 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -1,5 +1,8 @@ mod table_schema; +use in_memory_tree::VersionedHashMap; +use std::hash::Hash; + use self::table_schema::{ChangeHistorySchema, HistoryIndicesSchema, VersionedKeyValueSchema}; use crate::backends::TableReader; use crate::errors::Result; @@ -20,16 +23,16 @@ impl HistoryIndices { } } -struct PendingPart; +// struct PendingPart; -pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { - pending_part: PendingPart, +pub struct VersionedStore<'db, T: VersionedKeyValueSchema> where T::Key: Hash { + pending_part: VersionedHashMap, history_index_table: TableReader<'db, HistoryIndicesSchema>, commit_id_table: TableReader<'db, CommitIDSchema>, change_history_table: KeyValueBulks<'db, ChangeHistorySchema>, } -impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { +impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> where T::Key: Hash { pub fn get_pending_part(&self, commit: CommitID, key: &T::Key) -> Result> { todo!() } From b69c5e1f0b2490685e5160f215ddc65570d34cf5 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:07:40 +0800 Subject: [PATCH 03/79] convert error --- src/errors.rs | 6 ++++++ src/middlewares/versioned_flat_key_value/mod.rs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/errors.rs b/src/errors.rs index 00320f9..11f0e68 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,5 +1,8 @@ +use in_memory_tree::TreeError; use thiserror::Error; +use crate::middlewares::CommitID; + #[derive(Error, Debug)] pub enum StorageError { #[error("unknown version")] @@ -13,6 +16,9 @@ pub enum StorageError { #[error("decode error {0:?}")] DecodeError(#[from] DecodeError), + + #[error("tree error {0:?}")] + TreeError(#[from] TreeError), } pub type Result = ::std::result::Result; diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 6420b93..11c8832 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -34,7 +34,7 @@ pub struct VersionedStore<'db, T: VersionedKeyValueSchema> where T::Key: Hash { impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> where T::Key: Hash { pub fn get_pending_part(&self, commit: CommitID, key: &T::Key) -> Result> { - todo!() + self.pending_part.query(commit, key) } pub fn get_historical_part(&self, commit: CommitID, key: &T::Key) -> Result> { From 8f4ee438a452deb2d04c650198410e7ef98316de Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:12:50 +0800 Subject: [PATCH 04/79] impl get_pending_part --- src/errors.rs | 2 +- src/middlewares/versioned_flat_key_value/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 11f0e68..cb193a7 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -18,7 +18,7 @@ pub enum StorageError { DecodeError(#[from] DecodeError), #[error("tree error {0:?}")] - TreeError(#[from] TreeError), + TreeError(#[from] TreeError::), } pub type Result = ::std::result::Result; diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 11c8832..c7bf929 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -33,8 +33,8 @@ pub struct VersionedStore<'db, T: VersionedKeyValueSchema> where T::Key: Hash { } impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> where T::Key: Hash { - pub fn get_pending_part(&self, commit: CommitID, key: &T::Key) -> Result> { - self.pending_part.query(commit, key) + pub fn get_pending_part(&mut self, commit: CommitID, key: &T::Key) -> Result> { + Ok(self.pending_part.query(commit, key)?) } pub fn get_historical_part(&self, commit: CommitID, key: &T::Key) -> Result> { From 550bd4f09403d3dd2278d3d1b83d4025ba72d565 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 6 Sep 2024 19:14:13 +0800 Subject: [PATCH 05/79] impl add_to_pending_part --- src/middlewares/versioned_flat_key_value/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index c7bf929..19059c6 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -37,6 +37,12 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> where T::Key: Hash Ok(self.pending_part.query(commit, key)?) } + pub fn add_to_pending_part(&mut self, parent_commit: Option, commit: CommitID, + updates: Vec::<(T::Key, Option)>, + ) -> Result<()> { + Ok(self.pending_part.add_node(updates, commit, parent_commit)?) + } + pub fn get_historical_part(&self, commit: CommitID, key: &T::Key) -> Result> { let target_history_number = if let Some(value) = self.commit_id_table.get(&commit)? { value.into_owned() From 59da8c7ac2c55b3cbc9be62a71813b9a485da8c7 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 9 Sep 2024 10:43:49 +0800 Subject: [PATCH 06/79] rename pending_part --- Cargo.toml | 4 ++-- src/errors.rs | 4 ++-- .../Cargo.toml | 0 .../src/commit_tree.rs | 18 +++++++------- .../src/error.rs | 2 +- .../src/lib.rs | 2 +- .../src/versioned_hash_map.rs | 24 +++++++++---------- 7 files changed, 27 insertions(+), 27 deletions(-) rename src/{in_memory_tree => pending_part}/Cargo.toml (100%) rename src/{in_memory_tree => pending_part}/src/commit_tree.rs (94%) rename src/{in_memory_tree => pending_part}/src/error.rs (85%) rename src/{in_memory_tree => pending_part}/src/lib.rs (79%) rename src/{in_memory_tree => pending_part}/src/versioned_hash_map.rs (93%) diff --git a/Cargo.toml b/Cargo.toml index de8bcdc..44fe31b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,9 +12,9 @@ parking_lot = "0.12" ethereum-types = "0.12" -in-memory-tree = { path = "src/in_memory_tree" } +in-memory-tree = { path = "src/pending_part" } [workspace] members = [ - "src/in_memory_tree" + "src/pending_part" ] \ No newline at end of file diff --git a/src/errors.rs b/src/errors.rs index cb193a7..178ea45 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,4 +1,4 @@ -use in_memory_tree::TreeError; +use in_memory_tree::PendingError; use thiserror::Error; use crate::middlewares::CommitID; @@ -18,7 +18,7 @@ pub enum StorageError { DecodeError(#[from] DecodeError), #[error("tree error {0:?}")] - TreeError(#[from] TreeError::), + TreeError(#[from] PendingError::), } pub type Result = ::std::result::Result; diff --git a/src/in_memory_tree/Cargo.toml b/src/pending_part/Cargo.toml similarity index 100% rename from src/in_memory_tree/Cargo.toml rename to src/pending_part/Cargo.toml diff --git a/src/in_memory_tree/src/commit_tree.rs b/src/pending_part/src/commit_tree.rs similarity index 94% rename from src/in_memory_tree/src/commit_tree.rs rename to src/pending_part/src/commit_tree.rs index 790f049..d964fc7 100644 --- a/src/in_memory_tree/src/commit_tree.rs +++ b/src/pending_part/src/commit_tree.rs @@ -3,7 +3,7 @@ use std::{fmt::Debug, hash::Hash}; use slab::Slab; -use crate::TreeError; +use crate::PendingError; type SlabIndex = usize; @@ -38,11 +38,11 @@ impl fn commit_id_to_slab_index( &self, commit_id: CommitId, - ) -> Result> { + ) -> Result> { let slab_index = *self .index_map .get(&commit_id) - .ok_or_else(|| TreeError::CommitIDNotFound(commit_id))?; + .ok_or_else(|| PendingError::CommitIDNotFound(commit_id))?; Ok(slab_index) } @@ -68,7 +68,7 @@ impl commit_id: CommitId, parent_commit_id: Option, modifications: Vec<(Key, Option, Option)>, - ) -> Result<(), TreeError> { + ) -> Result<(), PendingError> { // return error if Some(parent_commit_id) but parent_commit_id does not exist let (parent_slab_index, parent_height) = if let Some(parent_commit_id) = parent_commit_id { let p_slab_index = self.commit_id_to_slab_index(parent_commit_id)?; @@ -77,13 +77,13 @@ impl } else { // return error if want to add root but there has been a root if self.has_root() { - return Err(TreeError::MultipleRootsNotAllowed); + return Err(PendingError::MultipleRootsNotAllowed); } (None, 0) }; // return error if commit_id exists if self.index_map.contains_key(&commit_id) { - return Err(TreeError::CommitIdAlreadyExists(commit_id)); + return Err(PendingError::CommitIdAlreadyExists(commit_id)); } let node = TreeNode::new(commit_id, parent_slab_index, parent_height, modifications); @@ -114,7 +114,7 @@ impl pub fn remove_subtree( &mut self, subroot_commit_id: CommitId, - ) -> Result<(), TreeError> { + ) -> Result<(), PendingError> { let subroot_slab_index = self.commit_id_to_slab_index(subroot_commit_id)?; let to_remove = self.bfs_subtree(subroot_slab_index); @@ -141,7 +141,7 @@ impl pub fn find_path( &self, target_commit_id: CommitId, - ) -> Result)>, TreeError> { + ) -> Result)>, PendingError> { let target_slab_index = self.commit_id_to_slab_index(target_commit_id)?; let mut target_node = self.slab_index_to_node(target_slab_index); let mut commits_rev = HashMap::new(); @@ -165,7 +165,7 @@ impl HashMap>, HashMap)>, ), - TreeError, + PendingError, > { let current_slab_index = self.commit_id_to_slab_index(current_commit_id)?; let target_slab_index = self.commit_id_to_slab_index(target_commit_id)?; diff --git a/src/in_memory_tree/src/error.rs b/src/pending_part/src/error.rs similarity index 85% rename from src/in_memory_tree/src/error.rs rename to src/pending_part/src/error.rs index 695ab6b..ca2e5ef 100644 --- a/src/in_memory_tree/src/error.rs +++ b/src/pending_part/src/error.rs @@ -3,7 +3,7 @@ use std::{fmt::Debug, hash::Hash}; use thiserror::Error; #[derive(Debug, PartialEq, Error)] -pub enum TreeError { +pub enum PendingError { #[error("commit id not found")] CommitIDNotFound(CommitId), #[error("multiple roots are not allowed")] diff --git a/src/in_memory_tree/src/lib.rs b/src/pending_part/src/lib.rs similarity index 79% rename from src/in_memory_tree/src/lib.rs rename to src/pending_part/src/lib.rs index 2cd91af..acdcb30 100644 --- a/src/in_memory_tree/src/lib.rs +++ b/src/pending_part/src/lib.rs @@ -2,5 +2,5 @@ pub mod commit_tree; pub mod error; pub mod versioned_hash_map; -pub use error::TreeError; +pub use error::PendingError; pub use versioned_hash_map::VersionedHashMap; diff --git a/src/in_memory_tree/src/versioned_hash_map.rs b/src/pending_part/src/versioned_hash_map.rs similarity index 93% rename from src/in_memory_tree/src/versioned_hash_map.rs rename to src/pending_part/src/versioned_hash_map.rs index 8217faf..b7ac563 100644 --- a/src/in_memory_tree/src/versioned_hash_map.rs +++ b/src/pending_part/src/versioned_hash_map.rs @@ -4,7 +4,7 @@ use std::{ hash::Hash, }; -use crate::{commit_tree::Tree, TreeError}; +use crate::{commit_tree::Tree, PendingError}; pub struct VersionedHashMap< Key: Eq + Hash + Clone + Ord, @@ -39,7 +39,7 @@ impl)>, commit_id: CommitId, parent_commit_id: Option, - ) -> Result<(), TreeError> { + ) -> Result<(), PendingError> { // let parent to be self.current self.walk_to_node(parent_commit_id)?; assert_eq!(parent_commit_id, self.current_node); @@ -67,7 +67,7 @@ impl Result, TreeError> { + ) -> Result, PendingError> { // let queried node to be self.current self.walk_to_node_unchecked(commit_id)?; assert_eq!(Some(commit_id), self.current_node); @@ -87,7 +87,7 @@ impl Result + 'a>, TreeError> { + ) -> Result + 'a>, PendingError> { // let queried node to be self.current self.walk_to_node_unchecked(commit_id)?; assert_eq!(Some(commit_id), self.current_node); @@ -112,7 +112,7 @@ impl, - ) -> Result<(), TreeError> { + ) -> Result<(), PendingError> { if target_commit_id.is_none() { self.current = BTreeMap::new(); self.current_node = None; @@ -124,7 +124,7 @@ impl Result<(), TreeError> { + ) -> Result<(), PendingError> { let (rollbacks, commits_rev) = if self.current_node.is_none() { let commits_rev = self.tree.find_path(target_commit_id)?; (HashMap::new(), commits_rev) @@ -257,11 +257,11 @@ mod tests { assert_eq!( forward_only_tree.add_node(1, None, Vec::new()), - Err(TreeError::MultipleRootsNotAllowed) + Err(PendingError::MultipleRootsNotAllowed) ); assert_eq!( versioned_hash_map.add_node(Vec::new(), 1, None), - Err(TreeError::MultipleRootsNotAllowed) + Err(PendingError::MultipleRootsNotAllowed) ); } @@ -272,11 +272,11 @@ mod tests { assert_eq!( forward_only_tree.add_node(1, Some(0), Vec::new()), - Err(TreeError::CommitIDNotFound(0)) + Err(PendingError::CommitIDNotFound(0)) ); assert_eq!( versioned_hash_map.add_node(Vec::new(), 1, Some(0)), - Err(TreeError::CommitIDNotFound(0)) + Err(PendingError::CommitIDNotFound(0)) ); } @@ -290,11 +290,11 @@ mod tests { assert_eq!( forward_only_tree.add_node(0, Some(0), Vec::new()), - Err(TreeError::CommitIdAlreadyExists(0)) + Err(PendingError::CommitIdAlreadyExists(0)) ); assert_eq!( versioned_hash_map.add_node(Vec::new(), 0, Some(0)), - Err(TreeError::CommitIdAlreadyExists(0)) + Err(PendingError::CommitIdAlreadyExists(0)) ); } } From 996b0d1f1bcba243b991c0c89337626dc9e77922 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:34:30 +0800 Subject: [PATCH 07/79] impl pending then history --- src/errors.rs | 4 +- .../versioned_flat_key_value/mod.rs | 14 ++++- src/pending_part/src/versioned_hash_map.rs | 52 ++++++------------- 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 178ea45..82e3c22 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -17,8 +17,8 @@ pub enum StorageError { #[error("decode error {0:?}")] DecodeError(#[from] DecodeError), - #[error("tree error {0:?}")] - TreeError(#[from] PendingError::), + #[error("pending error {0:?}")] + PendingError(#[from] PendingError::), } pub type Result = ::std::result::Result; diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 19059c6..e916d8c 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -34,7 +34,19 @@ pub struct VersionedStore<'db, T: VersionedKeyValueSchema> where T::Key: Hash { impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> where T::Key: Hash { pub fn get_pending_part(&mut self, commit: CommitID, key: &T::Key) -> Result> { - Ok(self.pending_part.query(commit, key)?) + let res_value = self.pending_part.query(commit, key); + let history_commit = match res_value { + Ok(None) => { self.pending_part.get_parent_of_root() }, + Err(in_memory_tree::PendingError::CommitIDNotFound(target_commit)) if target_commit == commit + => { Some(commit) }, + Ok(Some(value)) => { return Ok(value) }, + Err(e) => { return Err(StorageError::PendingError(e)) } + }; + if let Some(history_commit) = history_commit { + self.get_historical_part(history_commit, key) + } else { + Ok(None) + } } pub fn add_to_pending_part(&mut self, parent_commit: Option, commit: CommitID, diff --git a/src/pending_part/src/versioned_hash_map.rs b/src/pending_part/src/versioned_hash_map.rs index b7ac563..7b11304 100644 --- a/src/pending_part/src/versioned_hash_map.rs +++ b/src/pending_part/src/versioned_hash_map.rs @@ -11,6 +11,8 @@ pub struct VersionedHashMap< CommitId: Debug + Eq + Hash + Copy, Value: Clone, > { + parent_of_root: Option, + history: HashMap<(Key, CommitId), Option>, tree: Tree, @@ -21,8 +23,9 @@ pub struct VersionedHashMap< impl VersionedHashMap { - pub fn new() -> Self { + pub fn new(parent_of_root: Option) -> Self { VersionedHashMap { + parent_of_root, current: BTreeMap::new(), history: HashMap::new(), tree: Tree::new(), @@ -63,11 +66,14 @@ impl Result, PendingError> { + ) -> Result>, PendingError> { // let queried node to be self.current self.walk_to_node_unchecked(commit_id)?; assert_eq!(Some(commit_id), self.current_node); @@ -76,33 +82,11 @@ impl( - &'a mut self, - commit_id: CommitId, - key: Key, - ) -> Result + 'a>, PendingError> { - // let queried node to be self.current - self.walk_to_node_unchecked(commit_id)?; - assert_eq!(Some(commit_id), self.current_node); - // query - let range = self - .current - .range(key..) - .filter_map(|(key, (_, opt_value))| { - if let Some(value) = opt_value { - Some((key, value)) - } else { - None - } - }); - Ok(Box::new(range)) + pub fn get_parent_of_root(&self) -> Option { + self.parent_of_root } } @@ -197,7 +181,7 @@ mod tests { VersionedHashMap, ) { let mut forward_only_tree = Tree::new(); - let mut versioned_hash_map = VersionedHashMap::new(); + let mut versioned_hash_map = VersionedHashMap::new(None); for i in 1..=num_nodes as CommitId { let parent_commit_id = if i == 1 { @@ -236,11 +220,7 @@ mod tests { let result = versioned_hash_map.query(commit_id, &key).unwrap(); let current = forward_only_tree.find_path(commit_id).unwrap(); let answer = current.get(&key).and_then(|(_, value)| { - if value.is_some() { - Some(value.clone().unwrap()) - } else { - None - } + Some(value.clone()) }); assert_eq!(result, answer); } @@ -250,7 +230,7 @@ mod tests { #[test] fn test_multiple_roots_err() { let mut forward_only_tree = Tree::::new(); - let mut versioned_hash_map = VersionedHashMap::::new(); + let mut versioned_hash_map = VersionedHashMap::::new(None); forward_only_tree.add_node(0, None, Vec::new()).unwrap(); versioned_hash_map.add_node(Vec::new(), 0, None).unwrap(); @@ -268,7 +248,7 @@ mod tests { #[test] fn test_commit_id_not_found_err() { let mut forward_only_tree = Tree::::new(); - let mut versioned_hash_map = VersionedHashMap::::new(); + let mut versioned_hash_map = VersionedHashMap::::new(None); assert_eq!( forward_only_tree.add_node(1, Some(0), Vec::new()), @@ -283,7 +263,7 @@ mod tests { #[test] fn test_commit_id_already_exists_err() { let mut forward_only_tree = Tree::::new(); - let mut versioned_hash_map = VersionedHashMap::::new(); + let mut versioned_hash_map = VersionedHashMap::::new(None); forward_only_tree.add_node(0, None, Vec::new()).unwrap(); versioned_hash_map.add_node(Vec::new(), 0, None).unwrap(); From d03aaab63fbd95110e16d5750fbacce6d55c3bf7 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:27:57 +0800 Subject: [PATCH 08/79] fix get_historical_part --- src/middlewares/versioned_flat_key_value/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index e916d8c..9fa2379 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -77,7 +77,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> where T::Key: Hash Some(Ok((k, indices))) => { let KeyHistory(_, history_number) = k.as_ref(); let offset = target_history_number - history_number; - indices.as_ref().last(*history_number) + indices.as_ref().last(offset) } }; From aaddb8e49af9c4fe857179799a55ace37011a50c Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 9 Sep 2024 18:35:13 +0800 Subject: [PATCH 09/79] VersionedHashMap.history supports delete_commitid --- src/pending_part/src/versioned_hash_map.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/pending_part/src/versioned_hash_map.rs b/src/pending_part/src/versioned_hash_map.rs index 7b11304..3154254 100644 --- a/src/pending_part/src/versioned_hash_map.rs +++ b/src/pending_part/src/versioned_hash_map.rs @@ -13,7 +13,7 @@ pub struct VersionedHashMap< > { parent_of_root: Option, - history: HashMap<(Key, CommitId), Option>, + history: HashMap>>, tree: Tree, current: BTreeMap)>, @@ -49,7 +49,10 @@ impl { - if let Some(value) = self.history.get(&(key.clone(), old_commit_id)) { - self.current - .insert(key.clone(), (old_commit_id, value.clone())); - } else { - unreachable!( - "A modification recorded in a rollbacked commit is absent in history" - ); - } + let value = self.history.get(&old_commit_id).unwrap().get(&key).unwrap(); + self.current + .insert(key, (old_commit_id, value.clone())); } } } From 1986b52907a8dbb807a7cb46303c61e85c8fa1da Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 9 Sep 2024 18:54:21 +0800 Subject: [PATCH 10/79] Tree's rollbacks to support lazy delete --- src/pending_part/src/commit_tree.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/pending_part/src/commit_tree.rs b/src/pending_part/src/commit_tree.rs index d964fc7..0cddc97 100644 --- a/src/pending_part/src/commit_tree.rs +++ b/src/pending_part/src/commit_tree.rs @@ -35,6 +35,13 @@ impl } } + fn contains_commit_id( + &self, + commit_id: &CommitId, + ) -> bool { + self.index_map.contains_key(commit_id) + } + fn commit_id_to_slab_index( &self, commit_id: CommitId, @@ -187,6 +194,16 @@ impl target_node.target_up(&mut commits_rev); target_node = self.slab_index_to_node(target_node.parent.unwrap()); } + // check rollbacks' old_commit_id because TreeNodes are deleted + // in a lazy way with respect to TreeNodes.modifications + // todo: test this lazy method + for (_, old_commit_id_option) in rollbacks.iter_mut() { + if let Some(ref old_commit_id) = old_commit_id_option { + if !self.contains_commit_id(old_commit_id) { + *old_commit_id_option = None; + } + } + } // rollbacks or commits_rev may be empty, // they contain current and target (if they are not lca), respectively, // but they do not contain lca From cdc15f56fc29f12fefe755990ab6214a647aaad9 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 10 Sep 2024 12:56:08 +0800 Subject: [PATCH 11/79] impl change_root for pending's Tree --- src/pending_part/src/commit_tree.rs | 68 +++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/src/pending_part/src/commit_tree.rs b/src/pending_part/src/commit_tree.rs index 0cddc97..40e826c 100644 --- a/src/pending_part/src/commit_tree.rs +++ b/src/pending_part/src/commit_tree.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::{fmt::Debug, hash::Hash}; use slab::Slab; @@ -11,6 +11,9 @@ pub struct TreeNode, children: BTreeSet, + // todo: test lazy height + // height will not be changed even when root is changed + // height is only used for lca height: usize, commit_id: CommitId, @@ -101,8 +104,12 @@ impl } Ok(()) } +} - fn bfs_subtree(&mut self, subroot_slab_index: SlabIndex) -> Vec { +impl + Tree +{ + fn bfs_subtree(&self, subroot_slab_index: SlabIndex) -> Vec { let mut slab_indices = vec![subroot_slab_index]; let mut head = 0; while head < slab_indices.len() { @@ -118,27 +125,54 @@ impl slab_indices } - pub fn remove_subtree( + fn find_path_nodes( + &self, + target_slab_index: SlabIndex, + ) -> (Vec, HashSet) { + let mut target_node = self.slab_index_to_node(target_slab_index); + let mut path = Vec::new(); + let mut set = HashSet::new(); + while target_node.parent.is_some() { + let slab_index = target_node.parent.unwrap(); + set.insert(slab_index); + target_node = self.slab_index_to_node(slab_index); + path.push(target_node.commit_id); + } + (path, set) + } + + // todo: test + pub fn change_root( &mut self, - subroot_commit_id: CommitId, - ) -> Result<(), PendingError> { - let subroot_slab_index = self.commit_id_to_slab_index(subroot_commit_id)?; + commit_id: CommitId, + ) -> Result<(Vec, Vec), PendingError> { + let slab_index = self.commit_id_to_slab_index(commit_id)?; - let to_remove = self.bfs_subtree(subroot_slab_index); + // (root)..=(new_root's parent) + let (to_commit_rev, to_commit_set) = self.find_path_nodes(slab_index); + + // subtree of new_root + let to_maintain_vec = self.bfs_subtree(slab_index); + let to_maintain = BTreeSet::from_iter(to_maintain_vec.into_iter()); - // only subroot should update its parent, - // since, for other nodes to be removed, their parents will also be removed - if let Some(parent_slab_index) = self.slab_index_to_node(subroot_slab_index).parent { - let parent = &mut self.nodes[parent_slab_index]; - parent.children.remove(&subroot_slab_index); + // tree - subtree of new_root - (root)..-(new_root's parent) + let mut to_remove_indices = Vec::new(); + for (idx, _) in self.nodes.iter() { + if !to_maintain.contains(&idx) && !to_commit_set.contains(&idx) { + to_remove_indices.push(idx); + } } - - for &index in to_remove.iter() { - let node = self.nodes.remove(index); - self.index_map.remove(&node.commit_id); + let mut to_remove = Vec::new(); + for idx in to_remove_indices.into_iter() { + let to_remove_node = self.nodes.remove(idx); + self.index_map.remove(&to_remove_node.commit_id); + to_remove.push(to_remove_node.commit_id); } - Ok(()) + // set new_root's parent as None + self.nodes[slab_index].parent = None; + + Ok((to_commit_rev, to_remove)) } } From 5d2006c7556caa46042789cdffa51addf103d2a4 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 10 Sep 2024 15:07:49 +0800 Subject: [PATCH 12/79] impl change_root for Pending's VersionedHashMap --- src/pending_part/src/versioned_hash_map.rs | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/pending_part/src/versioned_hash_map.rs b/src/pending_part/src/versioned_hash_map.rs index 3154254..ba2137b 100644 --- a/src/pending_part/src/versioned_hash_map.rs +++ b/src/pending_part/src/versioned_hash_map.rs @@ -34,6 +34,32 @@ impl + VersionedHashMap +{ + pub fn change_root( + &mut self, + commit_id: CommitId, + ) -> Result>>>, PendingError> { + let (to_commit_rev, to_remove) = self.tree.change_root(commit_id)?; + if to_commit_rev.is_empty() { + assert!(to_remove.is_empty()); + return Ok(Vec::new()) + } + self.parent_of_root = Some(to_commit_rev[0]); + self.current.clear(); + self.current_node = None; + for to_remove_one in to_remove.into_iter() { + self.history.remove(&to_remove_one); + } + let mut to_commit = Vec::new(); + for to_commit_one in to_commit_rev.into_iter().rev() { + to_commit.push(self.history.remove(&to_commit_one)); + } + Ok(to_commit) + } +} + impl VersionedHashMap { From 0b4ea80fabb91b2cbe22a98eb024307fa5ac06a8 Mon Sep 17 00:00:00 2001 From: Chenxing Li Date: Tue, 10 Sep 2024 15:22:25 +0800 Subject: [PATCH 13/79] cargo fmt --- src/errors.rs | 4 +-- .../versioned_flat_key_value/mod.rs | 32 ++++++++++++------- src/pending_part/src/commit_tree.rs | 12 ++----- src/pending_part/src/versioned_hash_map.rs | 15 +++------ 4 files changed, 31 insertions(+), 32 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 82e3c22..2aafff2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -16,9 +16,9 @@ pub enum StorageError { #[error("decode error {0:?}")] DecodeError(#[from] DecodeError), - + #[error("pending error {0:?}")] - PendingError(#[from] PendingError::), + PendingError(#[from] PendingError), } pub type Result = ::std::result::Result; diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index af59958..5f04a06 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -1,7 +1,6 @@ mod serde; mod table_schema; - use in_memory_tree::VersionedHashMap; use std::hash::Hash; @@ -30,23 +29,31 @@ impl HistoryIndices { // struct PendingPart; - -pub struct VersionedStore<'db, T: VersionedKeyValueSchema> where T::Key: Hash { +pub struct VersionedStore<'db, T: VersionedKeyValueSchema> +where + T::Key: Hash, +{ pending_part: VersionedHashMap, history_index_table: TableReader<'db, HistoryIndicesTable>, commit_id_table: TableReader<'db, CommitIDSchema>, change_history_table: KeyValueStoreBulks<'db, HistoryChangeTable>, } -impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> where T::Key: Hash { +impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> +where + T::Key: Hash, +{ pub fn get_pending_part(&mut self, commit: CommitID, key: &T::Key) -> Result> { let res_value = self.pending_part.query(commit, key); let history_commit = match res_value { - Ok(None) => { self.pending_part.get_parent_of_root() }, - Err(in_memory_tree::PendingError::CommitIDNotFound(target_commit)) if target_commit == commit - => { Some(commit) }, - Ok(Some(value)) => { return Ok(value) }, - Err(e) => { return Err(StorageError::PendingError(e)) } + Ok(None) => self.pending_part.get_parent_of_root(), + Err(in_memory_tree::PendingError::CommitIDNotFound(target_commit)) + if target_commit == commit => + { + Some(commit) + } + Ok(Some(value)) => return Ok(value), + Err(e) => return Err(StorageError::PendingError(e)), }; if let Some(history_commit) = history_commit { self.get_historical_part(history_commit, key) @@ -55,8 +62,11 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> where T::Key: Hash } } - pub fn add_to_pending_part(&mut self, parent_commit: Option, commit: CommitID, - updates: Vec::<(T::Key, Option)>, + pub fn add_to_pending_part( + &mut self, + parent_commit: Option, + commit: CommitID, + updates: Vec<(T::Key, Option)>, ) -> Result<()> { Ok(self.pending_part.add_node(updates, commit, parent_commit)?) } diff --git a/src/pending_part/src/commit_tree.rs b/src/pending_part/src/commit_tree.rs index 40e826c..ed13484 100644 --- a/src/pending_part/src/commit_tree.rs +++ b/src/pending_part/src/commit_tree.rs @@ -38,10 +38,7 @@ impl } } - fn contains_commit_id( - &self, - commit_id: &CommitId, - ) -> bool { + fn contains_commit_id(&self, commit_id: &CommitId) -> bool { self.index_map.contains_key(commit_id) } @@ -125,10 +122,7 @@ impl slab_indices } - fn find_path_nodes( - &self, - target_slab_index: SlabIndex, - ) -> (Vec, HashSet) { + fn find_path_nodes(&self, target_slab_index: SlabIndex) -> (Vec, HashSet) { let mut target_node = self.slab_index_to_node(target_slab_index); let mut path = Vec::new(); let mut set = HashSet::new(); @@ -150,7 +144,7 @@ impl // (root)..=(new_root's parent) let (to_commit_rev, to_commit_set) = self.find_path_nodes(slab_index); - + // subtree of new_root let to_maintain_vec = self.bfs_subtree(slab_index); let to_maintain = BTreeSet::from_iter(to_maintain_vec.into_iter()); diff --git a/src/pending_part/src/versioned_hash_map.rs b/src/pending_part/src/versioned_hash_map.rs index ba2137b..fcff813 100644 --- a/src/pending_part/src/versioned_hash_map.rs +++ b/src/pending_part/src/versioned_hash_map.rs @@ -38,13 +38,13 @@ impl { pub fn change_root( - &mut self, + &mut self, commit_id: CommitId, ) -> Result>>>, PendingError> { let (to_commit_rev, to_remove) = self.tree.change_root(commit_id)?; if to_commit_rev.is_empty() { assert!(to_remove.is_empty()); - return Ok(Vec::new()) + return Ok(Vec::new()); } self.parent_of_root = Some(to_commit_rev[0]); self.current.clear(); @@ -75,9 +75,7 @@ impl { let value = self.history.get(&old_commit_id).unwrap().get(&key).unwrap(); - self.current - .insert(key, (old_commit_id, value.clone())); + self.current.insert(key, (old_commit_id, value.clone())); } } } @@ -243,9 +240,7 @@ mod tests { let key: Key = ikey.to_string().into_bytes(); let result = versioned_hash_map.query(commit_id, &key).unwrap(); let current = forward_only_tree.find_path(commit_id).unwrap(); - let answer = current.get(&key).and_then(|(_, value)| { - Some(value.clone()) - }); + let answer = current.get(&key).and_then(|(_, value)| Some(value.clone())); assert_eq!(result, answer); } } From 67d363e761f4213745219887e3d8aba78cf82920 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:07:58 +0800 Subject: [PATCH 14/79] move pending part to a module in the crate --- Cargo.toml | 9 ++++----- src/errors.rs | 3 +-- src/middlewares/mod.rs | 1 + src/middlewares/versioned_flat_key_value/mod.rs | 7 +++++-- .../pending_part}/commit_tree.rs | 2 +- .../versioned_flat_key_value/pending_part}/error.rs | 0 .../versioned_flat_key_value/pending_part/mod.rs} | 0 .../pending_part}/versioned_hash_map.rs | 2 +- src/pending_part/Cargo.toml | 12 ------------ 9 files changed, 13 insertions(+), 23 deletions(-) rename src/{pending_part/src => middlewares/versioned_flat_key_value/pending_part}/commit_tree.rs (99%) rename src/{pending_part/src => middlewares/versioned_flat_key_value/pending_part}/error.rs (100%) rename src/{pending_part/src/lib.rs => middlewares/versioned_flat_key_value/pending_part/mod.rs} (100%) rename src/{pending_part/src => middlewares/versioned_flat_key_value/pending_part}/versioned_hash_map.rs (99%) delete mode 100644 src/pending_part/Cargo.toml diff --git a/Cargo.toml b/Cargo.toml index 44fe31b..407da36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,9 +12,8 @@ parking_lot = "0.12" ethereum-types = "0.12" -in-memory-tree = { path = "src/pending_part" } +slab = "0.4.9" -[workspace] -members = [ - "src/pending_part" -] \ No newline at end of file +[dev-dependencies] +rand = "0.8.0" +rand_distr = "0.4.0" \ No newline at end of file diff --git a/src/errors.rs b/src/errors.rs index 2aafff2..8be05b7 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,7 +1,6 @@ -use in_memory_tree::PendingError; use thiserror::Error; -use crate::middlewares::CommitID; +use crate::middlewares::{CommitID, PendingError}; #[derive(Error, Debug)] pub enum StorageError { diff --git a/src/middlewares/mod.rs b/src/middlewares/mod.rs index 5d3c6a6..079ac26 100644 --- a/src/middlewares/mod.rs +++ b/src/middlewares/mod.rs @@ -6,3 +6,4 @@ pub use commit_id_schema::{ decode_history_number_rev, encode_history_number_rev, CommitID, CommitIDSchema, HistoryNumber, }; pub use key_value_store_bulks::{ChangeKey, KeyValueStoreBulks}; +pub use versioned_flat_key_value::PendingError; \ No newline at end of file diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 5f04a06..65d2ade 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -1,9 +1,12 @@ mod serde; mod table_schema; +mod pending_part; + +pub use pending_part::PendingError; -use in_memory_tree::VersionedHashMap; use std::hash::Hash; +use pending_part::VersionedHashMap; use self::table_schema::{HistoryChangeTable, HistoryIndicesTable, VersionedKeyValueSchema}; use super::ChangeKey; @@ -47,7 +50,7 @@ where let res_value = self.pending_part.query(commit, key); let history_commit = match res_value { Ok(None) => self.pending_part.get_parent_of_root(), - Err(in_memory_tree::PendingError::CommitIDNotFound(target_commit)) + Err(PendingError::CommitIDNotFound(target_commit)) if target_commit == commit => { Some(commit) diff --git a/src/pending_part/src/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs similarity index 99% rename from src/pending_part/src/commit_tree.rs rename to src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index ed13484..ac0fd8d 100644 --- a/src/pending_part/src/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -3,7 +3,7 @@ use std::{fmt::Debug, hash::Hash}; use slab::Slab; -use crate::PendingError; +use super::PendingError; type SlabIndex = usize; diff --git a/src/pending_part/src/error.rs b/src/middlewares/versioned_flat_key_value/pending_part/error.rs similarity index 100% rename from src/pending_part/src/error.rs rename to src/middlewares/versioned_flat_key_value/pending_part/error.rs diff --git a/src/pending_part/src/lib.rs b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs similarity index 100% rename from src/pending_part/src/lib.rs rename to src/middlewares/versioned_flat_key_value/pending_part/mod.rs diff --git a/src/pending_part/src/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs similarity index 99% rename from src/pending_part/src/versioned_hash_map.rs rename to src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index fcff813..c7ad564 100644 --- a/src/pending_part/src/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -4,7 +4,7 @@ use std::{ hash::Hash, }; -use crate::{commit_tree::Tree, PendingError}; +use super::{commit_tree::Tree, PendingError}; pub struct VersionedHashMap< Key: Eq + Hash + Clone + Ord, diff --git a/src/pending_part/Cargo.toml b/src/pending_part/Cargo.toml deleted file mode 100644 index 05d4337..0000000 --- a/src/pending_part/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "in-memory-tree" -version = "0.1.0" -edition = "2021" - -[dependencies] -slab = "0.4.9" -thiserror = "1" - -[dev-dependencies] -rand = "0.8.0" -rand_distr = "0.4.0" \ No newline at end of file From f046059e02ec29f87087a07fa42e0599003b076a Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:12:55 +0800 Subject: [PATCH 15/79] Add Hash to the VersionedKeyValueSchema::Key --- src/middlewares/versioned_flat_key_value/mod.rs | 12 ++---------- .../versioned_flat_key_value/table_schema.rs | 4 +++- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 65d2ade..c610802 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -4,8 +4,6 @@ mod pending_part; pub use pending_part::PendingError; -use std::hash::Hash; - use pending_part::VersionedHashMap; use self::table_schema::{HistoryChangeTable, HistoryIndicesTable, VersionedKeyValueSchema}; @@ -32,20 +30,14 @@ impl HistoryIndices { // struct PendingPart; -pub struct VersionedStore<'db, T: VersionedKeyValueSchema> -where - T::Key: Hash, -{ +pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { pending_part: VersionedHashMap, history_index_table: TableReader<'db, HistoryIndicesTable>, commit_id_table: TableReader<'db, CommitIDSchema>, change_history_table: KeyValueStoreBulks<'db, HistoryChangeTable>, } -impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> -where - T::Key: Hash, -{ +impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { pub fn get_pending_part(&mut self, commit: CommitID, key: &T::Key) -> Result> { let res_value = self.pending_part.query(commit, key); let history_commit = match res_value { diff --git a/src/middlewares/versioned_flat_key_value/table_schema.rs b/src/middlewares/versioned_flat_key_value/table_schema.rs index 86ec6bd..a80e03a 100644 --- a/src/middlewares/versioned_flat_key_value/table_schema.rs +++ b/src/middlewares/versioned_flat_key_value/table_schema.rs @@ -1,3 +1,5 @@ +use std::hash::Hash; + use crate::backends::{TableKey, TableName, TableSchema, TableValue, VersionedKVName}; use super::{HistoryChangeKey, HistoryIndexKey, HistoryIndices}; @@ -8,7 +10,7 @@ where HistoryIndexKey: TableKey, { const NAME: VersionedKVName; - type Key: TableKey + ToOwned + Clone; + type Key: TableKey + ToOwned + Clone + Hash; type Value: TableValue + Clone; } From 93ff8313234656da4d6566fa46f9f39a64a5fe25 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:22:45 +0800 Subject: [PATCH 16/79] target_commit != commit will not happen --- src/middlewares/versioned_flat_key_value/mod.rs | 4 ++-- .../versioned_flat_key_value/pending_part/commit_tree.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index c610802..7b2e768 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -42,9 +42,9 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { let res_value = self.pending_part.query(commit, key); let history_commit = match res_value { Ok(None) => self.pending_part.get_parent_of_root(), - Err(PendingError::CommitIDNotFound(target_commit)) - if target_commit == commit => + Err(PendingError::CommitIDNotFound(target_commit)) => { + assert_eq!(target_commit, commit); Some(commit) } Ok(Some(value)) => return Ok(value), diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index ac0fd8d..d3ec45e 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -202,7 +202,7 @@ impl ), PendingError, > { - let current_slab_index = self.commit_id_to_slab_index(current_commit_id)?; + let current_slab_index = self.commit_id_to_slab_index(current_commit_id).unwrap(); let target_slab_index = self.commit_id_to_slab_index(target_commit_id)?; let mut current_node = self.slab_index_to_node(current_slab_index); let mut target_node = self.slab_index_to_node(target_slab_index); From a70422956ed5338792b3a9a0b6412a9b76f73455 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:26:34 +0800 Subject: [PATCH 17/79] Adjust the orders in get_pending_part's match --- src/middlewares/versioned_flat_key_value/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 7b2e768..668393f 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -41,14 +41,14 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { pub fn get_pending_part(&mut self, commit: CommitID, key: &T::Key) -> Result> { let res_value = self.pending_part.query(commit, key); let history_commit = match res_value { + Ok(Some(value)) => return Ok(value), Ok(None) => self.pending_part.get_parent_of_root(), Err(PendingError::CommitIDNotFound(target_commit)) => { assert_eq!(target_commit, commit); Some(commit) } - Ok(Some(value)) => return Ok(value), - Err(e) => return Err(StorageError::PendingError(e)), + Err(other_err) => return Err(StorageError::PendingError(other_err)), }; if let Some(history_commit) = history_commit { self.get_historical_part(history_commit, key) From 327896c5d908655586d7e261215c1d76b72e5385 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:32:43 +0800 Subject: [PATCH 18/79] pending.add_node does not allow repeat keys --- .../versioned_flat_key_value/mod.rs | 7 +- .../pending_part/commit_tree.rs | 84 +++++----- .../pending_part/mod.rs | 1 + .../pending_part/versioned_hash_map.rs | 155 +++++++++--------- 4 files changed, 122 insertions(+), 125 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 668393f..e8a0aba 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -2,9 +2,12 @@ mod serde; mod table_schema; mod pending_part; +use std::collections::BTreeMap; + pub use pending_part::PendingError; use pending_part::VersionedHashMap; +use self::pending_part::pending_schema::PendingKeyValueConfig; use self::table_schema::{HistoryChangeTable, HistoryIndicesTable, VersionedKeyValueSchema}; use super::ChangeKey; @@ -31,7 +34,7 @@ impl HistoryIndices { // struct PendingPart; pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { - pending_part: VersionedHashMap, + pending_part: VersionedHashMap>, history_index_table: TableReader<'db, HistoryIndicesTable>, commit_id_table: TableReader<'db, CommitIDSchema>, change_history_table: KeyValueStoreBulks<'db, HistoryChangeTable>, @@ -61,7 +64,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { &mut self, parent_commit: Option, commit: CommitID, - updates: Vec<(T::Key, Option)>, + updates: BTreeMap>, ) -> Result<()> { Ok(self.pending_part.add_node(updates, commit, parent_commit)?) } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index d3ec45e..7449652 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -1,13 +1,13 @@ use std::collections::{BTreeSet, HashMap, HashSet}; -use std::{fmt::Debug, hash::Hash}; use slab::Slab; +use super::pending_schema::PendingKeyValueSchema; use super::PendingError; type SlabIndex = usize; -pub struct TreeNode { +pub struct TreeNode { parent: Option, children: BTreeSet, @@ -16,21 +16,19 @@ pub struct TreeNode, Option)>, + modifications: Vec<(S::Key, Option, Option)>, } -pub struct Tree { - nodes: Slab>, - index_map: HashMap, +pub struct Tree { + nodes: Slab>, + index_map: HashMap, } -impl - Tree -{ +impl Tree { pub fn new() -> Self { Tree { nodes: Slab::new(), @@ -38,14 +36,14 @@ impl } } - fn contains_commit_id(&self, commit_id: &CommitId) -> bool { + fn contains_commit_id(&self, commit_id: &S::CommitId) -> bool { self.index_map.contains_key(commit_id) } fn commit_id_to_slab_index( &self, - commit_id: CommitId, - ) -> Result> { + commit_id: S::CommitId, + ) -> Result> { let slab_index = *self .index_map .get(&commit_id) @@ -53,7 +51,7 @@ impl Ok(slab_index) } - fn slab_index_to_node(&self, slab_index: SlabIndex) -> &TreeNode { + fn slab_index_to_node(&self, slab_index: SlabIndex) -> &TreeNode { &self.nodes[slab_index] } @@ -61,21 +59,19 @@ impl !self.index_map.is_empty() } - pub fn get_parent_commit_id(&self, node: &TreeNode) -> Option { + pub fn get_parent_commit_id(&self, node: &TreeNode) -> Option { node.parent .and_then(|p_slab_index| Some(self.nodes[p_slab_index].commit_id)) } } -impl - Tree -{ +impl Tree { pub fn add_node( &mut self, - commit_id: CommitId, - parent_commit_id: Option, - modifications: Vec<(Key, Option, Option)>, - ) -> Result<(), PendingError> { + commit_id: S::CommitId, + parent_commit_id: Option, + modifications: Vec<(S::Key, Option, Option)>, + ) -> Result<(), PendingError> { // return error if Some(parent_commit_id) but parent_commit_id does not exist let (parent_slab_index, parent_height) = if let Some(parent_commit_id) = parent_commit_id { let p_slab_index = self.commit_id_to_slab_index(parent_commit_id)?; @@ -103,9 +99,7 @@ impl } } -impl - Tree -{ +impl Tree { fn bfs_subtree(&self, subroot_slab_index: SlabIndex) -> Vec { let mut slab_indices = vec![subroot_slab_index]; let mut head = 0; @@ -122,7 +116,7 @@ impl slab_indices } - fn find_path_nodes(&self, target_slab_index: SlabIndex) -> (Vec, HashSet) { + fn find_path_nodes(&self, target_slab_index: SlabIndex) -> (Vec, HashSet) { let mut target_node = self.slab_index_to_node(target_slab_index); let mut path = Vec::new(); let mut set = HashSet::new(); @@ -138,8 +132,8 @@ impl // todo: test pub fn change_root( &mut self, - commit_id: CommitId, - ) -> Result<(Vec, Vec), PendingError> { + commit_id: S::CommitId, + ) -> Result<(Vec, Vec), PendingError> { let slab_index = self.commit_id_to_slab_index(commit_id)?; // (root)..=(new_root's parent) @@ -170,13 +164,11 @@ impl } } -impl - Tree -{ +impl Tree { pub fn find_path( &self, - target_commit_id: CommitId, - ) -> Result)>, PendingError> { + target_commit_id: S::CommitId, + ) -> Result)>, PendingError> { let target_slab_index = self.commit_id_to_slab_index(target_commit_id)?; let mut target_node = self.slab_index_to_node(target_slab_index); let mut commits_rev = HashMap::new(); @@ -193,14 +185,14 @@ impl // correctness based on single root pub fn lca( &self, - current_commit_id: CommitId, - target_commit_id: CommitId, + current_commit_id: S::CommitId, + target_commit_id: S::CommitId, ) -> Result< ( - HashMap>, - HashMap)>, + HashMap>, + HashMap)>, ), - PendingError, + PendingError, > { let current_slab_index = self.commit_id_to_slab_index(current_commit_id).unwrap(); let target_slab_index = self.commit_id_to_slab_index(target_commit_id)?; @@ -239,14 +231,12 @@ impl } } -impl - TreeNode -{ +impl TreeNode { pub fn new( - commit_id: CommitId, + commit_id: S::CommitId, parent: Option, parent_height: usize, - modifications: Vec<(Key, Option, Option)>, + modifications: Vec<(S::Key, Option, Option)>, ) -> Self { Self { height: parent_height + 1, @@ -257,23 +247,23 @@ impl } } - pub fn get_commit_id(&self) -> CommitId { + pub fn get_commit_id(&self) -> S::CommitId { self.commit_id } pub fn get_modifications( &self, - ) -> impl Iterator, Option)> { + ) -> impl Iterator, Option)> { self.modifications.iter() } - pub fn current_up(&self, rollbacks: &mut HashMap>) { + pub fn current_up(&self, rollbacks: &mut HashMap>) { for (key, _, old_commit_id) in self.get_modifications() { rollbacks.insert(key.clone(), *old_commit_id); } } - pub fn target_up(&self, commits_rev: &mut HashMap)>) { + pub fn target_up(&self, commits_rev: &mut HashMap)>) { let commit_id = self.commit_id; for (key, value, _) in self.get_modifications() { commits_rev diff --git a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs index acdcb30..390ad70 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs @@ -1,6 +1,7 @@ pub mod commit_tree; pub mod error; pub mod versioned_hash_map; +pub mod pending_schema; pub use error::PendingError; pub use versioned_hash_map::VersionedHashMap; diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index c7ad564..8ac3103 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -1,29 +1,19 @@ -use std::{ - collections::{BTreeMap, HashMap}, - fmt::Debug, - hash::Hash, -}; - -use super::{commit_tree::Tree, PendingError}; - -pub struct VersionedHashMap< - Key: Eq + Hash + Clone + Ord, - CommitId: Debug + Eq + Hash + Copy, - Value: Clone, -> { - parent_of_root: Option, - - history: HashMap>>, - tree: Tree, - - current: BTreeMap)>, - current_node: Option, +use std::collections::{BTreeMap, HashMap}; + +use super::{commit_tree::Tree, pending_schema::PendingKeyValueSchema, PendingError}; + +pub struct VersionedHashMap { + parent_of_root: Option, + + history: HashMap>>, + tree: Tree, + + current: BTreeMap)>, + current_node: Option, } -impl - VersionedHashMap -{ - pub fn new(parent_of_root: Option) -> Self { +impl VersionedHashMap { + pub fn new(parent_of_root: Option) -> Self { VersionedHashMap { parent_of_root, current: BTreeMap::new(), @@ -34,13 +24,11 @@ impl - VersionedHashMap -{ +impl VersionedHashMap { pub fn change_root( &mut self, - commit_id: CommitId, - ) -> Result>>>, PendingError> { + commit_id: S::CommitId, + ) -> Result>>>, PendingError> { let (to_commit_rev, to_remove) = self.tree.change_root(commit_id)?; if to_commit_rev.is_empty() { assert!(to_remove.is_empty()); @@ -60,15 +48,13 @@ impl - VersionedHashMap -{ +impl VersionedHashMap { pub fn add_node( &mut self, - updates: Vec<(Key, Option)>, - commit_id: CommitId, - parent_commit_id: Option, - ) -> Result<(), PendingError> { + updates: BTreeMap>, + commit_id: S::CommitId, + parent_commit_id: Option, + ) -> Result<(), PendingError> { // let parent to be self.current self.walk_to_node(parent_commit_id)?; assert_eq!(parent_commit_id, self.current_node); @@ -90,6 +76,7 @@ impl Result>, PendingError> { + commit_id: S::CommitId, + key: &S::Key, + ) -> Result>, PendingError> { // let queried node to be self.current self.walk_to_node_unchecked(commit_id)?; assert_eq!(Some(commit_id), self.current_node); @@ -112,18 +99,16 @@ impl Option { + pub fn get_parent_of_root(&self) -> Option { self.parent_of_root } } -impl - VersionedHashMap -{ +impl VersionedHashMap { fn walk_to_node( &mut self, - target_commit_id: Option, - ) -> Result<(), PendingError> { + target_commit_id: Option, + ) -> Result<(), PendingError> { if target_commit_id.is_none() { self.current = BTreeMap::new(); self.current_node = None; @@ -134,8 +119,8 @@ impl Result<(), PendingError> { + target_commit_id: S::CommitId, + ) -> Result<(), PendingError> { let (rollbacks, commits_rev) = if self.current_node.is_none() { let commits_rev = self.tree.find_path(target_commit_id)?; (HashMap::new(), commits_rev) @@ -149,13 +134,13 @@ impl)>) { + fn commit_without_node_update(&mut self, commits_rev: HashMap)>) { for (key, (commit_id, value)) in commits_rev.into_iter() { self.current.insert(key, (commit_id, value)); } } - fn rollback_without_node_update(&mut self, rollbacks: HashMap>) { + fn rollback_without_node_update(&mut self, rollbacks: HashMap>) { for (key, old_commit_id) in rollbacks.into_iter() { match old_commit_id { None => { @@ -172,21 +157,32 @@ impl; - pub type Value = Vec; pub type CommitId = u64; - fn random_key_value(rng: &mut ThreadRng) -> (Key, Option) { + #[derive(Clone, Copy)] + struct TestSchema; + + impl VersionedKeyValueSchema for TestSchema { + const NAME: crate::backends::VersionedKVName = VersionedKVName::FlatKV; + type Key = u64; + type Value = u64; + } + + type TestPendingConfig = PendingKeyValueConfig; + + fn random_key_value(rng: &mut StdRng) -> (u64, Option) { // use a small key range to achieve more key conflicts let key_range = Uniform::from(0..10); - let key: Key = vec![key_range.sample(rng)]; - - let value: Option = if rng.gen_range(0..2) == 0 { - Some(vec![rng.gen::()]) + let key: u64 = key_range.sample(rng); + + let value: Option = if rng.gen_range(0..2) == 0 { + Some(rng.gen::()) } else { None }; @@ -196,10 +192,10 @@ mod tests { fn generate_random_tree( num_nodes: usize, - rng: &mut ThreadRng, + rng: &mut StdRng, ) -> ( - Tree, - VersionedHashMap, + Tree, + VersionedHashMap, ) { let mut forward_only_tree = Tree::new(); let mut versioned_hash_map = VersionedHashMap::new(None); @@ -210,9 +206,10 @@ mod tests { } else { Some(rng.gen_range(1..i)) }; - let mut updates = Vec::new(); + let mut updates = BTreeMap::new(); for _ in 0..5 { - updates.push(random_key_value(rng)); + let (key, value) = random_key_value(rng); + updates.insert(key, value); } let updates_none = updates .iter() @@ -231,13 +228,19 @@ mod tests { #[test] fn test_query() { - let num_nodes = 10; - let mut rng = rand::thread_rng(); + let num_nodes = 100; + + let seed: [u8; 32] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ]; + let mut rng = StdRng::from_seed(seed); + let (forward_only_tree, mut versioned_hash_map) = generate_random_tree(num_nodes, &mut rng); for _ in 0..100 { let commit_id = rng.gen_range(1..=num_nodes) as CommitId; for ikey in 0..10 { - let key: Key = ikey.to_string().into_bytes(); + let key: u64 = ikey; let result = versioned_hash_map.query(commit_id, &key).unwrap(); let current = forward_only_tree.find_path(commit_id).unwrap(); let answer = current.get(&key).and_then(|(_, value)| Some(value.clone())); @@ -248,51 +251,51 @@ mod tests { #[test] fn test_multiple_roots_err() { - let mut forward_only_tree = Tree::::new(); - let mut versioned_hash_map = VersionedHashMap::::new(None); + let mut forward_only_tree = Tree::::new(); + let mut versioned_hash_map = VersionedHashMap::::new(None); forward_only_tree.add_node(0, None, Vec::new()).unwrap(); - versioned_hash_map.add_node(Vec::new(), 0, None).unwrap(); + versioned_hash_map.add_node(BTreeMap::new(), 0, None).unwrap(); assert_eq!( forward_only_tree.add_node(1, None, Vec::new()), Err(PendingError::MultipleRootsNotAllowed) ); assert_eq!( - versioned_hash_map.add_node(Vec::new(), 1, None), + versioned_hash_map.add_node(BTreeMap::new(), 1, None), Err(PendingError::MultipleRootsNotAllowed) ); } #[test] fn test_commit_id_not_found_err() { - let mut forward_only_tree = Tree::::new(); - let mut versioned_hash_map = VersionedHashMap::::new(None); + let mut forward_only_tree = Tree::::new(); + let mut versioned_hash_map = VersionedHashMap::::new(None); assert_eq!( forward_only_tree.add_node(1, Some(0), Vec::new()), Err(PendingError::CommitIDNotFound(0)) ); assert_eq!( - versioned_hash_map.add_node(Vec::new(), 1, Some(0)), + versioned_hash_map.add_node(BTreeMap::new(), 1, Some(0)), Err(PendingError::CommitIDNotFound(0)) ); } #[test] fn test_commit_id_already_exists_err() { - let mut forward_only_tree = Tree::::new(); - let mut versioned_hash_map = VersionedHashMap::::new(None); + let mut forward_only_tree = Tree::::new(); + let mut versioned_hash_map = VersionedHashMap::::new(None); forward_only_tree.add_node(0, None, Vec::new()).unwrap(); - versioned_hash_map.add_node(Vec::new(), 0, None).unwrap(); + versioned_hash_map.add_node(BTreeMap::new(), 0, None).unwrap(); assert_eq!( forward_only_tree.add_node(0, Some(0), Vec::new()), Err(PendingError::CommitIdAlreadyExists(0)) ); assert_eq!( - versioned_hash_map.add_node(Vec::new(), 0, Some(0)), + versioned_hash_map.add_node(BTreeMap::new(), 0, Some(0)), Err(PendingError::CommitIdAlreadyExists(0)) ); } From 89be61fad2c3ace36291cf83dba43e7130aec6e4 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:46:19 +0800 Subject: [PATCH 19/79] use PendingKeyValueSchema & fix bug in add_node --- .../pending_part/pending_schema.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs new file mode 100644 index 0000000..9312c89 --- /dev/null +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -0,0 +1,23 @@ +use std::{fmt::Debug, hash::Hash, marker::PhantomData}; + +use crate::middlewares::versioned_flat_key_value::table_schema::VersionedKeyValueSchema; + +pub trait PendingKeyValueSchema { + type Key: Eq + Hash + Clone + Ord; + type CommitId: Debug + Eq + Hash + Copy; + type Value: Clone; +} + +pub struct PendingKeyValueConfig { + _marker: PhantomData<(T, CId)>, +} + +impl PendingKeyValueSchema for PendingKeyValueConfig +where + T: VersionedKeyValueSchema, + CId: Debug + Eq + Hash + Copy, +{ + type Key = T::Key; + type CommitId = CId; + type Value = T::Value; +} \ No newline at end of file From a8aa9cda7cc6d4841021b556d72eefce2ba8d0f8 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:49:27 +0800 Subject: [PATCH 20/79] rename xxx_to_xxx to get_xxx_by_xxx --- .../pending_part/commit_tree.rs | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 7449652..e9e31a9 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -40,7 +40,7 @@ impl Tree { self.index_map.contains_key(commit_id) } - fn commit_id_to_slab_index( + fn get_slab_index_by_commit_id( &self, commit_id: S::CommitId, ) -> Result> { @@ -51,7 +51,7 @@ impl Tree { Ok(slab_index) } - fn slab_index_to_node(&self, slab_index: SlabIndex) -> &TreeNode { + fn get_node_by_slab_index(&self, slab_index: SlabIndex) -> &TreeNode { &self.nodes[slab_index] } @@ -74,8 +74,8 @@ impl Tree { ) -> Result<(), PendingError> { // return error if Some(parent_commit_id) but parent_commit_id does not exist let (parent_slab_index, parent_height) = if let Some(parent_commit_id) = parent_commit_id { - let p_slab_index = self.commit_id_to_slab_index(parent_commit_id)?; - let p_height = self.slab_index_to_node(p_slab_index).height; + let p_slab_index = self.get_slab_index_by_commit_id(parent_commit_id)?; + let p_height = self.get_node_by_slab_index(p_slab_index).height; (Some(p_slab_index), p_height) } else { // return error if want to add root but there has been a root @@ -104,7 +104,7 @@ impl Tree { let mut slab_indices = vec![subroot_slab_index]; let mut head = 0; while head < slab_indices.len() { - let node = self.slab_index_to_node(slab_indices[head]); + let node = self.get_node_by_slab_index(slab_indices[head]); for &child_index in &node.children { slab_indices.push(child_index); @@ -117,13 +117,13 @@ impl Tree { } fn find_path_nodes(&self, target_slab_index: SlabIndex) -> (Vec, HashSet) { - let mut target_node = self.slab_index_to_node(target_slab_index); + let mut target_node = self.get_node_by_slab_index(target_slab_index); let mut path = Vec::new(); let mut set = HashSet::new(); while target_node.parent.is_some() { let slab_index = target_node.parent.unwrap(); set.insert(slab_index); - target_node = self.slab_index_to_node(slab_index); + target_node = self.get_node_by_slab_index(slab_index); path.push(target_node.commit_id); } (path, set) @@ -134,7 +134,7 @@ impl Tree { &mut self, commit_id: S::CommitId, ) -> Result<(Vec, Vec), PendingError> { - let slab_index = self.commit_id_to_slab_index(commit_id)?; + let slab_index = self.get_slab_index_by_commit_id(commit_id)?; // (root)..=(new_root's parent) let (to_commit_rev, to_commit_set) = self.find_path_nodes(slab_index); @@ -169,15 +169,15 @@ impl Tree { &self, target_commit_id: S::CommitId, ) -> Result)>, PendingError> { - let target_slab_index = self.commit_id_to_slab_index(target_commit_id)?; - let mut target_node = self.slab_index_to_node(target_slab_index); + let target_slab_index = self.get_slab_index_by_commit_id(target_commit_id)?; + let mut target_node = self.get_node_by_slab_index(target_slab_index); let mut commits_rev = HashMap::new(); loop { target_node.target_up(&mut commits_rev); if target_node.parent.is_none() { break; } - target_node = self.slab_index_to_node(target_node.parent.unwrap()); + target_node = self.get_node_by_slab_index(target_node.parent.unwrap()); } Ok(commits_rev) } @@ -194,25 +194,25 @@ impl Tree { ), PendingError, > { - let current_slab_index = self.commit_id_to_slab_index(current_commit_id).unwrap(); - let target_slab_index = self.commit_id_to_slab_index(target_commit_id)?; - let mut current_node = self.slab_index_to_node(current_slab_index); - let mut target_node = self.slab_index_to_node(target_slab_index); + let current_slab_index = self.get_slab_index_by_commit_id(current_commit_id).unwrap(); + let target_slab_index = self.get_slab_index_by_commit_id(target_commit_id)?; + let mut current_node = self.get_node_by_slab_index(current_slab_index); + let mut target_node = self.get_node_by_slab_index(target_slab_index); let mut rollbacks = HashMap::new(); let mut commits_rev = HashMap::new(); while current_node.height > target_node.height { current_node.current_up(&mut rollbacks); - current_node = self.slab_index_to_node(current_node.parent.unwrap()); + current_node = self.get_node_by_slab_index(current_node.parent.unwrap()); } while target_node.height > current_node.height { target_node.target_up(&mut commits_rev); - target_node = self.slab_index_to_node(target_node.parent.unwrap()); + target_node = self.get_node_by_slab_index(target_node.parent.unwrap()); } while current_node.commit_id != target_node.commit_id { current_node.current_up(&mut rollbacks); - current_node = self.slab_index_to_node(current_node.parent.unwrap()); + current_node = self.get_node_by_slab_index(current_node.parent.unwrap()); target_node.target_up(&mut commits_rev); - target_node = self.slab_index_to_node(target_node.parent.unwrap()); + target_node = self.get_node_by_slab_index(target_node.parent.unwrap()); } // check rollbacks' old_commit_id because TreeNodes are deleted // in a lazy way with respect to TreeNodes.modifications From 2873c51a415a2e6e9cef1d51f9d1c91fa0e400fc Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 10:40:02 +0800 Subject: [PATCH 21/79] while let Some(parent_slab_index) --- .../versioned_flat_key_value/pending_part/commit_tree.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index e9e31a9..286eb78 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -120,10 +120,9 @@ impl Tree { let mut target_node = self.get_node_by_slab_index(target_slab_index); let mut path = Vec::new(); let mut set = HashSet::new(); - while target_node.parent.is_some() { - let slab_index = target_node.parent.unwrap(); - set.insert(slab_index); - target_node = self.get_node_by_slab_index(slab_index); + while let Some(parent_slab_index) = target_node.parent { + set.insert(parent_slab_index); + target_node = self.get_node_by_slab_index(parent_slab_index); path.push(target_node.commit_id); } (path, set) From 82c48dd7d4c6f819e61432f3f6c645eef9a2ba64 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 10:46:59 +0800 Subject: [PATCH 22/79] while let Some(parent_slab_index) in find_path --- .../versioned_flat_key_value/pending_part/commit_tree.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 286eb78..0289dfd 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -171,12 +171,10 @@ impl Tree { let target_slab_index = self.get_slab_index_by_commit_id(target_commit_id)?; let mut target_node = self.get_node_by_slab_index(target_slab_index); let mut commits_rev = HashMap::new(); - loop { + target_node.target_up(&mut commits_rev); + while let Some(parent_slab_index) = target_node.parent { + target_node = self.get_node_by_slab_index(parent_slab_index); target_node.target_up(&mut commits_rev); - if target_node.parent.is_none() { - break; - } - target_node = self.get_node_by_slab_index(target_node.parent.unwrap()); } Ok(commits_rev) } From a62363e7d6f79f125ff242069923900efac6e79c Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 10:53:18 +0800 Subject: [PATCH 23/79] impl get_slab_index_by_commit_id --- .../pending_part/commit_tree.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 0289dfd..c5606d7 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -55,6 +55,14 @@ impl Tree { &self.nodes[slab_index] } + fn get_node_by_commit_id( + &self, + commit_id: S::CommitId + ) -> Result<&TreeNode, PendingError> { + let slab_index = self.get_slab_index_by_commit_id(commit_id)?; + Ok(self.get_node_by_slab_index(slab_index)) + } + fn has_root(&self) -> bool { !self.index_map.is_empty() } @@ -168,8 +176,7 @@ impl Tree { &self, target_commit_id: S::CommitId, ) -> Result)>, PendingError> { - let target_slab_index = self.get_slab_index_by_commit_id(target_commit_id)?; - let mut target_node = self.get_node_by_slab_index(target_slab_index); + let mut target_node = self.get_node_by_commit_id(target_commit_id)?; let mut commits_rev = HashMap::new(); target_node.target_up(&mut commits_rev); while let Some(parent_slab_index) = target_node.parent { @@ -191,10 +198,8 @@ impl Tree { ), PendingError, > { - let current_slab_index = self.get_slab_index_by_commit_id(current_commit_id).unwrap(); - let target_slab_index = self.get_slab_index_by_commit_id(target_commit_id)?; - let mut current_node = self.get_node_by_slab_index(current_slab_index); - let mut target_node = self.get_node_by_slab_index(target_slab_index); + let mut current_node = self.get_node_by_commit_id(current_commit_id).unwrap(); + let mut target_node = self.get_node_by_commit_id(target_commit_id)?; let mut rollbacks = HashMap::new(); let mut commits_rev = HashMap::new(); while current_node.height > target_node.height { From 61aefca279f2fd853d6028e54751481e9fec52d6 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:13:38 +0800 Subject: [PATCH 24/79] rename current_up to export_rollback_data --- .../pending_part/commit_tree.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index c5606d7..c60d789 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -178,10 +178,10 @@ impl Tree { ) -> Result)>, PendingError> { let mut target_node = self.get_node_by_commit_id(target_commit_id)?; let mut commits_rev = HashMap::new(); - target_node.target_up(&mut commits_rev); + target_node.export_commit_data(&mut commits_rev); while let Some(parent_slab_index) = target_node.parent { target_node = self.get_node_by_slab_index(parent_slab_index); - target_node.target_up(&mut commits_rev); + target_node.export_commit_data(&mut commits_rev); } Ok(commits_rev) } @@ -203,17 +203,17 @@ impl Tree { let mut rollbacks = HashMap::new(); let mut commits_rev = HashMap::new(); while current_node.height > target_node.height { - current_node.current_up(&mut rollbacks); + current_node.export_rollback_data(&mut rollbacks); current_node = self.get_node_by_slab_index(current_node.parent.unwrap()); } while target_node.height > current_node.height { - target_node.target_up(&mut commits_rev); + target_node.export_commit_data(&mut commits_rev); target_node = self.get_node_by_slab_index(target_node.parent.unwrap()); } while current_node.commit_id != target_node.commit_id { - current_node.current_up(&mut rollbacks); + current_node.export_rollback_data(&mut rollbacks); current_node = self.get_node_by_slab_index(current_node.parent.unwrap()); - target_node.target_up(&mut commits_rev); + target_node.export_commit_data(&mut commits_rev); target_node = self.get_node_by_slab_index(target_node.parent.unwrap()); } // check rollbacks' old_commit_id because TreeNodes are deleted @@ -259,13 +259,13 @@ impl TreeNode { self.modifications.iter() } - pub fn current_up(&self, rollbacks: &mut HashMap>) { + pub fn export_rollback_data(&self, rollbacks: &mut HashMap>) { for (key, _, old_commit_id) in self.get_modifications() { rollbacks.insert(key.clone(), *old_commit_id); } } - pub fn target_up(&self, commits_rev: &mut HashMap)>) { + pub fn export_commit_data(&self, commits_rev: &mut HashMap)>) { let commit_id = self.commit_id; for (key, value, _) in self.get_modifications() { commits_rev From c4feb4921d2f7b12f597c9b4ced547fd8d6265bf Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:16:51 +0800 Subject: [PATCH 25/79] derive Eq for PendingError --- src/middlewares/versioned_flat_key_value/pending_part/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/error.rs b/src/middlewares/versioned_flat_key_value/pending_part/error.rs index ca2e5ef..295b891 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/error.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/error.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, hash::Hash}; use thiserror::Error; -#[derive(Debug, PartialEq, Error)] +#[derive(Debug, PartialEq, Error, Eq)] pub enum PendingError { #[error("commit id not found")] CommitIDNotFound(CommitId), From 81ab0c8cb9cf5b91db454290ad29a5d4fd3846b0 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:36:28 +0800 Subject: [PATCH 26/79] Move history_inner_map outside the loop. --- .../pending_part/versioned_hash_map.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 8ac3103..0160866 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -60,8 +60,8 @@ impl VersionedHashMap { assert_eq!(parent_commit_id, self.current_node); // add node let mut modifications = Vec::new(); + let mut history_inner_map = HashMap::new(); for (key, value) in updates.into_iter() { - let history_inner_map = self.history.entry(commit_id).or_insert_with(HashMap::new); history_inner_map.insert(key.clone(), value.clone()); let old_commit_id = { if let Some((old_commit_id, _)) = @@ -74,6 +74,9 @@ impl VersionedHashMap { }; modifications.push((key, value, old_commit_id)); } + // PendingError::CommitIdAlreadyExists(commit_id) will be reported + // in self.tree.add_node + self.history.entry(commit_id).or_insert_with(|| history_inner_map); self.tree .add_node(commit_id, parent_commit_id, modifications)?; self.current_node = Some(commit_id); From ce0df7f7e5fae7d25c9000f17b62076687761430 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:06:48 +0800 Subject: [PATCH 27/79] remove unnecessary bracket --- .../pending_part/versioned_hash_map.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 0160866..cc3ac50 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -63,15 +63,14 @@ impl VersionedHashMap { let mut history_inner_map = HashMap::new(); for (key, value) in updates.into_iter() { history_inner_map.insert(key.clone(), value.clone()); - let old_commit_id = { + let old_commit_id = if let Some((old_commit_id, _)) = self.current.insert(key.clone(), (commit_id, value.clone())) { Some(old_commit_id) } else { None - } - }; + }; modifications.push((key, value, old_commit_id)); } // PendingError::CommitIdAlreadyExists(commit_id) will be reported From b20b6c344d098056f6b7d896e299ab86c3d54b24 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:16:51 +0800 Subject: [PATCH 28/79] early report CommitIdAlreadyExists in add_node --- .../pending_part/versioned_hash_map.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index cc3ac50..30b0516 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -55,14 +55,17 @@ impl VersionedHashMap { commit_id: S::CommitId, parent_commit_id: Option, ) -> Result<(), PendingError> { + if self.history.contains_key(&commit_id) { + return Err(PendingError::CommitIdAlreadyExists(commit_id)); + } // let parent to be self.current self.walk_to_node(parent_commit_id)?; assert_eq!(parent_commit_id, self.current_node); // add node let mut modifications = Vec::new(); - let mut history_inner_map = HashMap::new(); + let mut inner_map = HashMap::new(); for (key, value) in updates.into_iter() { - history_inner_map.insert(key.clone(), value.clone()); + inner_map.insert(key.clone(), value.clone()); let old_commit_id = if let Some((old_commit_id, _)) = self.current.insert(key.clone(), (commit_id, value.clone())) @@ -73,9 +76,7 @@ impl VersionedHashMap { }; modifications.push((key, value, old_commit_id)); } - // PendingError::CommitIdAlreadyExists(commit_id) will be reported - // in self.tree.add_node - self.history.entry(commit_id).or_insert_with(|| history_inner_map); + self.history.insert(commit_id, inner_map); self.tree .add_node(commit_id, parent_commit_id, modifications)?; self.current_node = Some(commit_id); From cadccf402fcb5caa25ac345feffd716242eeebed Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:05:02 +0800 Subject: [PATCH 29/79] use if let Some(x) = xx {} else {} instd of unwrap --- .../pending_part/commit_tree.rs | 2 +- .../pending_part/versioned_hash_map.rs | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index c60d789..7c9b2ce 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -187,7 +187,7 @@ impl Tree { } // correctness based on single root - pub fn lca( + pub(super) fn lca( &self, current_commit_id: S::CommitId, target_commit_id: S::CommitId, diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 30b0516..344365b 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -112,11 +112,11 @@ impl VersionedHashMap { &mut self, target_commit_id: Option, ) -> Result<(), PendingError> { - if target_commit_id.is_none() { + if let Some(target_commit_id) = target_commit_id { + self.walk_to_node_unchecked(target_commit_id)?; + } else { self.current = BTreeMap::new(); self.current_node = None; - } else { - self.walk_to_node_unchecked(target_commit_id.unwrap())?; } Ok(()) } @@ -124,13 +124,13 @@ impl VersionedHashMap { &mut self, target_commit_id: S::CommitId, ) -> Result<(), PendingError> { - let (rollbacks, commits_rev) = if self.current_node.is_none() { - let commits_rev = self.tree.find_path(target_commit_id)?; - (HashMap::new(), commits_rev) - } else { - self.tree - .lca(self.current_node.unwrap(), target_commit_id)? - }; + let (rollbacks, commits_rev) = + if let Some(current_commit_id) = self.current_node { + self.tree + .lca(current_commit_id, target_commit_id)? + } else { + (HashMap::new(), self.tree.find_path(target_commit_id)?) + }; self.rollback_without_node_update(rollbacks); self.commit_without_node_update(commits_rev); self.current_node = Some(target_commit_id); From 68c0e7a480f252fcb8c6dfa49ac1040b38547e6d Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:20:52 +0800 Subject: [PATCH 30/79] impl get_parent_node() --- .../pending_part/commit_tree.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 7c9b2ce..f9134bf 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -71,6 +71,11 @@ impl Tree { node.parent .and_then(|p_slab_index| Some(self.nodes[p_slab_index].commit_id)) } + + fn get_parent_node(&self, node: &TreeNode) -> Option<&TreeNode> { + node.parent + .and_then(|p_slab_index| Some(self.get_node_by_slab_index(p_slab_index))) + } } impl Tree { @@ -204,17 +209,17 @@ impl Tree { let mut commits_rev = HashMap::new(); while current_node.height > target_node.height { current_node.export_rollback_data(&mut rollbacks); - current_node = self.get_node_by_slab_index(current_node.parent.unwrap()); + current_node = self.get_parent_node(current_node).unwrap(); } while target_node.height > current_node.height { target_node.export_commit_data(&mut commits_rev); - target_node = self.get_node_by_slab_index(target_node.parent.unwrap()); + target_node = self.get_parent_node(target_node).unwrap(); } while current_node.commit_id != target_node.commit_id { current_node.export_rollback_data(&mut rollbacks); - current_node = self.get_node_by_slab_index(current_node.parent.unwrap()); + current_node = self.get_parent_node(current_node).unwrap(); target_node.export_commit_data(&mut commits_rev); - target_node = self.get_node_by_slab_index(target_node.parent.unwrap()); + target_node = self.get_parent_node(target_node).unwrap(); } // check rollbacks' old_commit_id because TreeNodes are deleted // in a lazy way with respect to TreeNodes.modifications From 54f9245c2c2adf33cb168982f27997f84609130c Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:21:13 +0800 Subject: [PATCH 31/79] fix cargo clippy --- .../pending_part/commit_tree.rs | 30 ++++++++----------- .../pending_part/pending_schema.rs | 14 ++++++++- .../pending_part/versioned_hash_map.rs | 14 ++++----- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index f9134bf..c67f6f1 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use slab::Slab; -use super::pending_schema::PendingKeyValueSchema; +use super::pending_schema::{CIdVecPair, Commits, Modifications, PendingKeyValueSchema, RollComm}; use super::PendingError; type SlabIndex = usize; @@ -20,7 +20,7 @@ pub struct TreeNode { // before current node, the old value of this key is modified by which commit_id, // if none, this key is absent before current node // here must use CommitID instead of SlabIndex (which may be reused, see slab doc) - modifications: Vec<(S::Key, Option, Option)>, + modifications: Modifications, } pub struct Tree { @@ -47,7 +47,7 @@ impl Tree { let slab_index = *self .index_map .get(&commit_id) - .ok_or_else(|| PendingError::CommitIDNotFound(commit_id))?; + .ok_or(PendingError::CommitIDNotFound(commit_id))?; Ok(slab_index) } @@ -69,12 +69,12 @@ impl Tree { pub fn get_parent_commit_id(&self, node: &TreeNode) -> Option { node.parent - .and_then(|p_slab_index| Some(self.nodes[p_slab_index].commit_id)) + .map(|p_slab_index| self.nodes[p_slab_index].commit_id) } fn get_parent_node(&self, node: &TreeNode) -> Option<&TreeNode> { node.parent - .and_then(|p_slab_index| Some(self.get_node_by_slab_index(p_slab_index))) + .map(|p_slab_index| self.get_node_by_slab_index(p_slab_index)) } } @@ -83,7 +83,7 @@ impl Tree { &mut self, commit_id: S::CommitId, parent_commit_id: Option, - modifications: Vec<(S::Key, Option, Option)>, + modifications: Modifications, ) -> Result<(), PendingError> { // return error if Some(parent_commit_id) but parent_commit_id does not exist let (parent_slab_index, parent_height) = if let Some(parent_commit_id) = parent_commit_id { @@ -145,7 +145,7 @@ impl Tree { pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> Result<(Vec, Vec), PendingError> { + ) -> Result, PendingError> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; // (root)..=(new_root's parent) @@ -153,7 +153,7 @@ impl Tree { // subtree of new_root let to_maintain_vec = self.bfs_subtree(slab_index); - let to_maintain = BTreeSet::from_iter(to_maintain_vec.into_iter()); + let to_maintain = BTreeSet::from_iter(to_maintain_vec); // tree - subtree of new_root - (root)..-(new_root's parent) let mut to_remove_indices = Vec::new(); @@ -180,7 +180,7 @@ impl Tree { pub fn find_path( &self, target_commit_id: S::CommitId, - ) -> Result)>, PendingError> { + ) -> Result, PendingError> { let mut target_node = self.get_node_by_commit_id(target_commit_id)?; let mut commits_rev = HashMap::new(); target_node.export_commit_data(&mut commits_rev); @@ -196,13 +196,7 @@ impl Tree { &self, current_commit_id: S::CommitId, target_commit_id: S::CommitId, - ) -> Result< - ( - HashMap>, - HashMap)>, - ), - PendingError, - > { + ) -> Result, PendingError> { let mut current_node = self.get_node_by_commit_id(current_commit_id).unwrap(); let mut target_node = self.get_node_by_commit_id(target_commit_id)?; let mut rollbacks = HashMap::new(); @@ -243,7 +237,7 @@ impl TreeNode { commit_id: S::CommitId, parent: Option, parent_height: usize, - modifications: Vec<(S::Key, Option, Option)>, + modifications: Modifications, ) -> Self { Self { height: parent_height + 1, @@ -270,7 +264,7 @@ impl TreeNode { } } - pub fn export_commit_data(&self, commits_rev: &mut HashMap)>) { + pub fn export_commit_data(&self, commits_rev: &mut Commits) { let commit_id = self.commit_id; for (key, value, _) in self.get_modifications() { commits_rev diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 9312c89..4c38fa8 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, hash::Hash, marker::PhantomData}; +use std::{collections::{BTreeMap, HashMap}, fmt::Debug, hash::Hash, marker::PhantomData}; use crate::middlewares::versioned_flat_key_value::table_schema::VersionedKeyValueSchema; @@ -8,6 +8,18 @@ pub trait PendingKeyValueSchema { type Value: Clone; } +pub type Modifications = Vec<(::Key, Option<::Value>, Option<::CommitId>)>; +pub type Commits = HashMap< + ::Key, + (::CommitId, Option<::Value>) +>; +pub type Rollbacks = HashMap<::Key, Option<::CommitId>>; +pub type History = HashMap<::CommitId, HashMap<::Key, Option<::Value>>>; +pub type Current = BTreeMap<::Key, (::CommitId, Option<::Value>)>; +pub type ToCommit = Vec<(::CommitId, Option::Key, Option<::Value>>>)>; +pub type CIdVecPair = (Vec<::CommitId>, Vec<::CommitId>); +pub type RollComm = (Rollbacks, Commits); + pub struct PendingKeyValueConfig { _marker: PhantomData<(T, CId)>, } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 344365b..2703d56 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -1,14 +1,14 @@ use std::collections::{BTreeMap, HashMap}; -use super::{commit_tree::Tree, pending_schema::PendingKeyValueSchema, PendingError}; +use super::{commit_tree::Tree, pending_schema::{Commits, Current, History, PendingKeyValueSchema, ToCommit}, PendingError}; pub struct VersionedHashMap { parent_of_root: Option, - history: HashMap>>, + history: History, tree: Tree, - current: BTreeMap)>, + current: Current, current_node: Option, } @@ -28,7 +28,7 @@ impl VersionedHashMap { pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> Result>>>, PendingError> { + ) -> Result, PendingError> { let (to_commit_rev, to_remove) = self.tree.change_root(commit_id)?; if to_commit_rev.is_empty() { assert!(to_remove.is_empty()); @@ -42,7 +42,7 @@ impl VersionedHashMap { } let mut to_commit = Vec::new(); for to_commit_one in to_commit_rev.into_iter().rev() { - to_commit.push(self.history.remove(&to_commit_one)); + to_commit.push((to_commit_one, self.history.remove(&to_commit_one))); } Ok(to_commit) } @@ -98,7 +98,7 @@ impl VersionedHashMap { let value = self .current .get(key) - .and_then(|(_, value)| Some(value.clone())); + .map(|(_, value)| value.clone()); Ok(value) } @@ -137,7 +137,7 @@ impl VersionedHashMap { Ok(()) } - fn commit_without_node_update(&mut self, commits_rev: HashMap)>) { + fn commit_without_node_update(&mut self, commits_rev: Commits) { for (key, (commit_id, value)) in commits_rev.into_iter() { self.current.insert(key, (commit_id, value)); } From 61fbf5a873221b0c9c7091efc0c6c20649208e79 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:23:02 +0800 Subject: [PATCH 32/79] cargo fmt --- src/middlewares/mod.rs | 2 +- .../versioned_flat_key_value/mod.rs | 7 +- .../pending_part/commit_tree.rs | 7 +- .../pending_part/mod.rs | 2 +- .../pending_part/pending_schema.rs | 46 ++++++++++--- .../pending_part/versioned_hash_map.rs | 67 ++++++++++--------- 6 files changed, 83 insertions(+), 48 deletions(-) diff --git a/src/middlewares/mod.rs b/src/middlewares/mod.rs index 079ac26..876e73b 100644 --- a/src/middlewares/mod.rs +++ b/src/middlewares/mod.rs @@ -6,4 +6,4 @@ pub use commit_id_schema::{ decode_history_number_rev, encode_history_number_rev, CommitID, CommitIDSchema, HistoryNumber, }; pub use key_value_store_bulks::{ChangeKey, KeyValueStoreBulks}; -pub use versioned_flat_key_value::PendingError; \ No newline at end of file +pub use versioned_flat_key_value::PendingError; diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index e8a0aba..2454e91 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -1,14 +1,14 @@ +mod pending_part; mod serde; mod table_schema; -mod pending_part; use std::collections::BTreeMap; pub use pending_part::PendingError; -use pending_part::VersionedHashMap; use self::pending_part::pending_schema::PendingKeyValueConfig; use self::table_schema::{HistoryChangeTable, HistoryIndicesTable, VersionedKeyValueSchema}; +use pending_part::VersionedHashMap; use super::ChangeKey; use super::CommitIDSchema; @@ -46,8 +46,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { let history_commit = match res_value { Ok(Some(value)) => return Ok(value), Ok(None) => self.pending_part.get_parent_of_root(), - Err(PendingError::CommitIDNotFound(target_commit)) => - { + Err(PendingError::CommitIDNotFound(target_commit)) => { assert_eq!(target_commit, commit); Some(commit) } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index c67f6f1..f1476d2 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -57,7 +57,7 @@ impl Tree { fn get_node_by_commit_id( &self, - commit_id: S::CommitId + commit_id: S::CommitId, ) -> Result<&TreeNode, PendingError> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; Ok(self.get_node_by_slab_index(slab_index)) @@ -129,7 +129,10 @@ impl Tree { slab_indices } - fn find_path_nodes(&self, target_slab_index: SlabIndex) -> (Vec, HashSet) { + fn find_path_nodes( + &self, + target_slab_index: SlabIndex, + ) -> (Vec, HashSet) { let mut target_node = self.get_node_by_slab_index(target_slab_index); let mut path = Vec::new(); let mut set = HashSet::new(); diff --git a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs index 390ad70..e76a045 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs @@ -1,7 +1,7 @@ pub mod commit_tree; pub mod error; -pub mod versioned_hash_map; pub mod pending_schema; +pub mod versioned_hash_map; pub use error::PendingError; pub use versioned_hash_map::VersionedHashMap; diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 4c38fa8..20198d1 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -1,4 +1,9 @@ -use std::{collections::{BTreeMap, HashMap}, fmt::Debug, hash::Hash, marker::PhantomData}; +use std::{ + collections::{BTreeMap, HashMap}, + fmt::Debug, + hash::Hash, + marker::PhantomData, +}; use crate::middlewares::versioned_flat_key_value::table_schema::VersionedKeyValueSchema; @@ -8,16 +13,39 @@ pub trait PendingKeyValueSchema { type Value: Clone; } -pub type Modifications = Vec<(::Key, Option<::Value>, Option<::CommitId>)>; +pub type Modifications = Vec<( + ::Key, + Option<::Value>, + Option<::CommitId>, +)>; pub type Commits = HashMap< ::Key, - (::CommitId, Option<::Value>) + ( + ::CommitId, + Option<::Value>, + ), +>; +pub type Rollbacks = + HashMap<::Key, Option<::CommitId>>; +pub type History = HashMap< + ::CommitId, + HashMap<::Key, Option<::Value>>, >; -pub type Rollbacks = HashMap<::Key, Option<::CommitId>>; -pub type History = HashMap<::CommitId, HashMap<::Key, Option<::Value>>>; -pub type Current = BTreeMap<::Key, (::CommitId, Option<::Value>)>; -pub type ToCommit = Vec<(::CommitId, Option::Key, Option<::Value>>>)>; -pub type CIdVecPair = (Vec<::CommitId>, Vec<::CommitId>); +pub type Current = BTreeMap< + ::Key, + ( + ::CommitId, + Option<::Value>, + ), +>; +pub type ToCommit = Vec<( + ::CommitId, + Option::Key, Option<::Value>>>, +)>; +pub type CIdVecPair = ( + Vec<::CommitId>, + Vec<::CommitId>, +); pub type RollComm = (Rollbacks, Commits); pub struct PendingKeyValueConfig { @@ -32,4 +60,4 @@ where type Key = T::Key; type CommitId = CId; type Value = T::Value; -} \ No newline at end of file +} diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 2703d56..91fe8fc 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -1,6 +1,10 @@ use std::collections::{BTreeMap, HashMap}; -use super::{commit_tree::Tree, pending_schema::{Commits, Current, History, PendingKeyValueSchema, ToCommit}, PendingError}; +use super::{ + commit_tree::Tree, + pending_schema::{Commits, Current, History, PendingKeyValueSchema, ToCommit}, + PendingError, +}; pub struct VersionedHashMap { parent_of_root: Option, @@ -66,14 +70,13 @@ impl VersionedHashMap { let mut inner_map = HashMap::new(); for (key, value) in updates.into_iter() { inner_map.insert(key.clone(), value.clone()); - let old_commit_id = - if let Some((old_commit_id, _)) = - self.current.insert(key.clone(), (commit_id, value.clone())) - { - Some(old_commit_id) - } else { - None - }; + let old_commit_id = if let Some((old_commit_id, _)) = + self.current.insert(key.clone(), (commit_id, value.clone())) + { + Some(old_commit_id) + } else { + None + }; modifications.push((key, value, old_commit_id)); } self.history.insert(commit_id, inner_map); @@ -95,10 +98,7 @@ impl VersionedHashMap { self.walk_to_node_unchecked(commit_id)?; assert_eq!(Some(commit_id), self.current_node); // query - let value = self - .current - .get(key) - .map(|(_, value)| value.clone()); + let value = self.current.get(key).map(|(_, value)| value.clone()); Ok(value) } @@ -124,13 +124,11 @@ impl VersionedHashMap { &mut self, target_commit_id: S::CommitId, ) -> Result<(), PendingError> { - let (rollbacks, commits_rev) = - if let Some(current_commit_id) = self.current_node { - self.tree - .lca(current_commit_id, target_commit_id)? - } else { - (HashMap::new(), self.tree.find_path(target_commit_id)?) - }; + let (rollbacks, commits_rev) = if let Some(current_commit_id) = self.current_node { + self.tree.lca(current_commit_id, target_commit_id)? + } else { + (HashMap::new(), self.tree.find_path(target_commit_id)?) + }; self.rollback_without_node_update(rollbacks); self.commit_without_node_update(commits_rev); self.current_node = Some(target_commit_id); @@ -160,7 +158,13 @@ impl VersionedHashMap { #[cfg(test)] mod tests { - use crate::{backends::VersionedKVName, middlewares::versioned_flat_key_value::{pending_part::pending_schema::PendingKeyValueConfig, table_schema::VersionedKeyValueSchema}}; + use crate::{ + backends::VersionedKVName, + middlewares::versioned_flat_key_value::{ + pending_part::pending_schema::PendingKeyValueConfig, + table_schema::VersionedKeyValueSchema, + }, + }; use super::*; use rand::{rngs::StdRng, Rng, SeedableRng}; @@ -173,7 +177,7 @@ mod tests { impl VersionedKeyValueSchema for TestSchema { const NAME: crate::backends::VersionedKVName = VersionedKVName::FlatKV; - type Key = u64; + type Key = u64; type Value = u64; } @@ -183,7 +187,7 @@ mod tests { // use a small key range to achieve more key conflicts let key_range = Uniform::from(0..10); let key: u64 = key_range.sample(rng); - + let value: Option = if rng.gen_range(0..2) == 0 { Some(rng.gen::()) } else { @@ -196,10 +200,7 @@ mod tests { fn generate_random_tree( num_nodes: usize, rng: &mut StdRng, - ) -> ( - Tree, - VersionedHashMap, - ) { + ) -> (Tree, VersionedHashMap) { let mut forward_only_tree = Tree::new(); let mut versioned_hash_map = VersionedHashMap::new(None); @@ -234,8 +235,8 @@ mod tests { let num_nodes = 100; let seed: [u8; 32] = [ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, ]; let mut rng = StdRng::from_seed(seed); @@ -258,7 +259,9 @@ mod tests { let mut versioned_hash_map = VersionedHashMap::::new(None); forward_only_tree.add_node(0, None, Vec::new()).unwrap(); - versioned_hash_map.add_node(BTreeMap::new(), 0, None).unwrap(); + versioned_hash_map + .add_node(BTreeMap::new(), 0, None) + .unwrap(); assert_eq!( forward_only_tree.add_node(1, None, Vec::new()), @@ -291,7 +294,9 @@ mod tests { let mut versioned_hash_map = VersionedHashMap::::new(None); forward_only_tree.add_node(0, None, Vec::new()).unwrap(); - versioned_hash_map.add_node(BTreeMap::new(), 0, None).unwrap(); + versioned_hash_map + .add_node(BTreeMap::new(), 0, None) + .unwrap(); assert_eq!( forward_only_tree.add_node(0, Some(0), Vec::new()), From 707d9a5db000f8b43faffc4c925d1b843a67616b Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:22:31 +0800 Subject: [PATCH 33/79] define Key then use it to define others --- .../pending_part/pending_schema.rs | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 20198d1..4f2b593 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -13,38 +13,42 @@ pub trait PendingKeyValueSchema { type Value: Clone; } +pub type Key = ::Key; +pub type Value = ::Value; +pub type CommitId = ::CommitId; + pub type Modifications = Vec<( - ::Key, - Option<::Value>, - Option<::CommitId>, + Key, + Option>, + Option>, )>; pub type Commits = HashMap< - ::Key, + Key, ( - ::CommitId, - Option<::Value>, + CommitId, + Option>, ), >; pub type Rollbacks = - HashMap<::Key, Option<::CommitId>>; + HashMap, Option>>; pub type History = HashMap< - ::CommitId, - HashMap<::Key, Option<::Value>>, + CommitId, + HashMap, Option>>, >; pub type Current = BTreeMap< - ::Key, + Key, ( - ::CommitId, - Option<::Value>, + CommitId, + Option>, ), >; pub type ToCommit = Vec<( - ::CommitId, - Option::Key, Option<::Value>>>, + CommitId, + Option, Option>>>, )>; pub type CIdVecPair = ( - Vec<::CommitId>, - Vec<::CommitId>, + Vec>, + Vec>, ); pub type RollComm = (Rollbacks, Commits); From 311bcedb42b9bc2d79f01df0d80c35788e16f5b8 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:36:35 +0800 Subject: [PATCH 34/79] cargo fmt --- .../pending_part/pending_schema.rs | 40 ++++--------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 4f2b593..663973b 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -17,39 +17,13 @@ pub type Key = ::Key; pub type Value = ::Value; pub type CommitId = ::CommitId; -pub type Modifications = Vec<( - Key, - Option>, - Option>, -)>; -pub type Commits = HashMap< - Key, - ( - CommitId, - Option>, - ), ->; -pub type Rollbacks = - HashMap, Option>>; -pub type History = HashMap< - CommitId, - HashMap, Option>>, ->; -pub type Current = BTreeMap< - Key, - ( - CommitId, - Option>, - ), ->; -pub type ToCommit = Vec<( - CommitId, - Option, Option>>>, -)>; -pub type CIdVecPair = ( - Vec>, - Vec>, -); +pub type Modifications = Vec<(Key, Option>, Option>)>; +pub type Commits = HashMap, (CommitId, Option>)>; +pub type Rollbacks = HashMap, Option>>; +pub type History = HashMap, HashMap, Option>>>; +pub type Current = BTreeMap, (CommitId, Option>)>; +pub type ToCommit = Vec<(CommitId, Option, Option>>>)>; +pub type CIdVecPair = (Vec>, Vec>); pub type RollComm = (Rollbacks, Commits); pub struct PendingKeyValueConfig { From 69aa823c5f6a30901009d51733979e9c9865710c Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:11:21 +0800 Subject: [PATCH 35/79] allow(clippy::type_complexity) History --- .../versioned_flat_key_value/pending_part/pending_schema.rs | 1 - .../pending_part/versioned_hash_map.rs | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 663973b..8e1f2eb 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -20,7 +20,6 @@ pub type CommitId = ::CommitId; pub type Modifications = Vec<(Key, Option>, Option>)>; pub type Commits = HashMap, (CommitId, Option>)>; pub type Rollbacks = HashMap, Option>>; -pub type History = HashMap, HashMap, Option>>>; pub type Current = BTreeMap, (CommitId, Option>)>; pub type ToCommit = Vec<(CommitId, Option, Option>>>)>; pub type CIdVecPair = (Vec>, Vec>); diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 91fe8fc..47e969a 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -2,14 +2,15 @@ use std::collections::{BTreeMap, HashMap}; use super::{ commit_tree::Tree, - pending_schema::{Commits, Current, History, PendingKeyValueSchema, ToCommit}, + pending_schema::{Commits, Current, PendingKeyValueSchema, ToCommit}, PendingError, }; pub struct VersionedHashMap { parent_of_root: Option, - history: History, + #[allow(clippy::type_complexity)] + history: HashMap>>, tree: Tree, current: Current, From 26967d048de2204d7e221f17200beb6ce5209773 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:12:59 +0800 Subject: [PATCH 36/79] allow(clippy::type_complexity) Current --- .../versioned_flat_key_value/pending_part/pending_schema.rs | 3 +-- .../pending_part/versioned_hash_map.rs | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 8e1f2eb..4d0e57c 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashMap}, + collections::HashMap, fmt::Debug, hash::Hash, marker::PhantomData, @@ -20,7 +20,6 @@ pub type CommitId = ::CommitId; pub type Modifications = Vec<(Key, Option>, Option>)>; pub type Commits = HashMap, (CommitId, Option>)>; pub type Rollbacks = HashMap, Option>>; -pub type Current = BTreeMap, (CommitId, Option>)>; pub type ToCommit = Vec<(CommitId, Option, Option>>>)>; pub type CIdVecPair = (Vec>, Vec>); pub type RollComm = (Rollbacks, Commits); diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 47e969a..6538d29 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashMap}; use super::{ commit_tree::Tree, - pending_schema::{Commits, Current, PendingKeyValueSchema, ToCommit}, + pending_schema::{Commits, PendingKeyValueSchema, ToCommit}, PendingError, }; @@ -13,7 +13,8 @@ pub struct VersionedHashMap { history: HashMap>>, tree: Tree, - current: Current, + #[allow(clippy::type_complexity)] + current: BTreeMap)>, current_node: Option, } From 413103cc9073f38406e00f81a6a9d651e57cb44d Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:14:34 +0800 Subject: [PATCH 37/79] allow(clippy::type_complexity) ToCommit --- .../pending_part/pending_schema.rs | 8 +------- .../pending_part/versioned_hash_map.rs | 8 ++++++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 4d0e57c..181ac03 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -1,9 +1,4 @@ -use std::{ - collections::HashMap, - fmt::Debug, - hash::Hash, - marker::PhantomData, -}; +use std::{collections::HashMap, fmt::Debug, hash::Hash, marker::PhantomData}; use crate::middlewares::versioned_flat_key_value::table_schema::VersionedKeyValueSchema; @@ -20,7 +15,6 @@ pub type CommitId = ::CommitId; pub type Modifications = Vec<(Key, Option>, Option>)>; pub type Commits = HashMap, (CommitId, Option>)>; pub type Rollbacks = HashMap, Option>>; -pub type ToCommit = Vec<(CommitId, Option, Option>>>)>; pub type CIdVecPair = (Vec>, Vec>); pub type RollComm = (Rollbacks, Commits); diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 6538d29..3ac14b3 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashMap}; use super::{ commit_tree::Tree, - pending_schema::{Commits, PendingKeyValueSchema, ToCommit}, + pending_schema::{Commits, PendingKeyValueSchema}, PendingError, }; @@ -31,10 +31,14 @@ impl VersionedHashMap { } impl VersionedHashMap { + #[allow(clippy::type_complexity)] pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> Result, PendingError> { + ) -> Result< + Vec<(S::CommitId, Option>>)>, + PendingError, + > { let (to_commit_rev, to_remove) = self.tree.change_root(commit_id)?; if to_commit_rev.is_empty() { assert!(to_remove.is_empty()); From 9b49ebd536b22ed505efbd3d7404ba97e7306f29 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:16:25 +0800 Subject: [PATCH 38/79] allow(clippy::type_complexity) CIdVevPair --- .../versioned_flat_key_value/pending_part/commit_tree.rs | 5 +++-- .../versioned_flat_key_value/pending_part/pending_schema.rs | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index f1476d2..50fd55a 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use slab::Slab; -use super::pending_schema::{CIdVecPair, Commits, Modifications, PendingKeyValueSchema, RollComm}; +use super::pending_schema::{Commits, Modifications, PendingKeyValueSchema, RollComm}; use super::PendingError; type SlabIndex = usize; @@ -145,10 +145,11 @@ impl Tree { } // todo: test + #[allow(clippy::type_complexity)] pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> Result, PendingError> { + ) -> Result<(Vec, Vec), PendingError> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; // (root)..=(new_root's parent) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 181ac03..713e54a 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -15,7 +15,6 @@ pub type CommitId = ::CommitId; pub type Modifications = Vec<(Key, Option>, Option>)>; pub type Commits = HashMap, (CommitId, Option>)>; pub type Rollbacks = HashMap, Option>>; -pub type CIdVecPair = (Vec>, Vec>); pub type RollComm = (Rollbacks, Commits); pub struct PendingKeyValueConfig { From 90d12eae774abc5c32b9842c1b5861ac9de0ae22 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:19:32 +0800 Subject: [PATCH 39/79] allow(clippy::type_complexity) RollComm --- .../versioned_flat_key_value/pending_part/commit_tree.rs | 5 +++-- .../versioned_flat_key_value/pending_part/pending_schema.rs | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 50fd55a..63f4e4b 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use slab::Slab; -use super::pending_schema::{Commits, Modifications, PendingKeyValueSchema, RollComm}; +use super::pending_schema::{Commits, Modifications, PendingKeyValueSchema}; use super::PendingError; type SlabIndex = usize; @@ -196,11 +196,12 @@ impl Tree { } // correctness based on single root + #[allow(clippy::type_complexity)] pub(super) fn lca( &self, current_commit_id: S::CommitId, target_commit_id: S::CommitId, - ) -> Result, PendingError> { + ) -> Result<(HashMap>, Commits), PendingError> { let mut current_node = self.get_node_by_commit_id(current_commit_id).unwrap(); let mut target_node = self.get_node_by_commit_id(target_commit_id)?; let mut rollbacks = HashMap::new(); diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 713e54a..d2ad3a1 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -14,8 +14,6 @@ pub type CommitId = ::CommitId; pub type Modifications = Vec<(Key, Option>, Option>)>; pub type Commits = HashMap, (CommitId, Option>)>; -pub type Rollbacks = HashMap, Option>>; -pub type RollComm = (Rollbacks, Commits); pub struct PendingKeyValueConfig { _marker: PhantomData<(T, CId)>, From e02e1d7a4aabc6c1c33e204527c913afaaede52b Mon Sep 17 00:00:00 2001 From: Chenxing Li Date: Fri, 13 Sep 2024 15:37:49 +0800 Subject: [PATCH 40/79] Rewrite current maintainence --- .../versioned_flat_key_value/mod.rs | 2 +- .../pending_part/commit_tree.rs | 66 +++---- .../pending_part/pending_schema.rs | 33 +++- .../pending_part/versioned_hash_map.rs | 187 ++++++++++-------- 4 files changed, 160 insertions(+), 128 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 2454e91..002e082 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -2,7 +2,7 @@ mod pending_part; mod serde; mod table_schema; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; pub use pending_part::PendingError; diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 63f4e4b..8366e75 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -1,8 +1,11 @@ -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use slab::Slab; -use super::pending_schema::{Commits, Modifications, PendingKeyValueSchema}; +use super::pending_schema::{ + ApplyMap, ApplyRecord, CommitIdVec, PendingKeyValueSchema, RecoverMap, RecoverRecord, + Result as PendResult, +}; use super::PendingError; type SlabIndex = usize; @@ -20,7 +23,7 @@ pub struct TreeNode { // before current node, the old value of this key is modified by which commit_id, // if none, this key is absent before current node // here must use CommitID instead of SlabIndex (which may be reused, see slab doc) - modifications: Modifications, + modifications: RecoverMap, } pub struct Tree { @@ -40,10 +43,7 @@ impl Tree { self.index_map.contains_key(commit_id) } - fn get_slab_index_by_commit_id( - &self, - commit_id: S::CommitId, - ) -> Result> { + fn get_slab_index_by_commit_id(&self, commit_id: S::CommitId) -> PendResult { let slab_index = *self .index_map .get(&commit_id) @@ -55,10 +55,7 @@ impl Tree { &self.nodes[slab_index] } - fn get_node_by_commit_id( - &self, - commit_id: S::CommitId, - ) -> Result<&TreeNode, PendingError> { + fn get_node_by_commit_id(&self, commit_id: S::CommitId) -> PendResult<&TreeNode, S> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; Ok(self.get_node_by_slab_index(slab_index)) } @@ -83,8 +80,8 @@ impl Tree { &mut self, commit_id: S::CommitId, parent_commit_id: Option, - modifications: Modifications, - ) -> Result<(), PendingError> { + modifications: RecoverMap, + ) -> PendResult<(), S> { // return error if Some(parent_commit_id) but parent_commit_id does not exist let (parent_slab_index, parent_height) = if let Some(parent_commit_id) = parent_commit_id { let p_slab_index = self.get_slab_index_by_commit_id(parent_commit_id)?; @@ -132,7 +129,7 @@ impl Tree { fn find_path_nodes( &self, target_slab_index: SlabIndex, - ) -> (Vec, HashSet) { + ) -> (CommitIdVec, HashSet) { let mut target_node = self.get_node_by_slab_index(target_slab_index); let mut path = Vec::new(); let mut set = HashSet::new(); @@ -145,11 +142,10 @@ impl Tree { } // todo: test - #[allow(clippy::type_complexity)] pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> Result<(Vec, Vec), PendingError> { + ) -> PendResult<(CommitIdVec, CommitIdVec), S> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; // (root)..=(new_root's parent) @@ -181,12 +177,12 @@ impl Tree { } impl Tree { - pub fn find_path( + pub fn get_apply_map_from_root( &self, target_commit_id: S::CommitId, - ) -> Result, PendingError> { + ) -> PendResult, S> { let mut target_node = self.get_node_by_commit_id(target_commit_id)?; - let mut commits_rev = HashMap::new(); + let mut commits_rev = BTreeMap::new(); target_node.export_commit_data(&mut commits_rev); while let Some(parent_slab_index) = target_node.parent { target_node = self.get_node_by_slab_index(parent_slab_index); @@ -196,16 +192,15 @@ impl Tree { } // correctness based on single root - #[allow(clippy::type_complexity)] - pub(super) fn lca( + pub(super) fn collect_rollback_and_apply_ops( &self, current_commit_id: S::CommitId, target_commit_id: S::CommitId, - ) -> Result<(HashMap>, Commits), PendingError> { + ) -> PendResult<(BTreeMap>, ApplyMap), S> { let mut current_node = self.get_node_by_commit_id(current_commit_id).unwrap(); let mut target_node = self.get_node_by_commit_id(target_commit_id)?; - let mut rollbacks = HashMap::new(); - let mut commits_rev = HashMap::new(); + let mut rollbacks = BTreeMap::new(); + let mut commits_rev = BTreeMap::new(); while current_node.height > target_node.height { current_node.export_rollback_data(&mut rollbacks); current_node = self.get_parent_node(current_node).unwrap(); @@ -242,7 +237,7 @@ impl TreeNode { commit_id: S::CommitId, parent: Option, parent_height: usize, - modifications: Modifications, + modifications: RecoverMap, ) -> Self { Self { height: parent_height + 1, @@ -257,24 +252,25 @@ impl TreeNode { self.commit_id } - pub fn get_modifications( - &self, - ) -> impl Iterator, Option)> { - self.modifications.iter() + pub fn get_modifications(&self) -> &RecoverMap { + &self.modifications } - pub fn export_rollback_data(&self, rollbacks: &mut HashMap>) { - for (key, _, old_commit_id) in self.get_modifications() { - rollbacks.insert(key.clone(), *old_commit_id); + pub fn export_rollback_data(&self, rollbacks: &mut BTreeMap>) { + for (key, RecoverRecord { last_commit_id, .. }) in self.modifications.iter() { + rollbacks.insert(key.clone(), *last_commit_id); } } - pub fn export_commit_data(&self, commits_rev: &mut Commits) { + pub fn export_commit_data(&self, commits_rev: &mut ApplyMap) { let commit_id = self.commit_id; - for (key, value, _) in self.get_modifications() { + for (key, RecoverRecord { value, .. }) in self.modifications.iter() { commits_rev .entry(key.clone()) - .or_insert_with(|| (commit_id, value.clone())); + .or_insert_with(|| ApplyRecord { + commit_id, + value: value.clone(), + }); } } } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index d2ad3a1..0ba0ba7 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -1,19 +1,40 @@ -use std::{collections::HashMap, fmt::Debug, hash::Hash, marker::PhantomData}; +use std::{ + collections::{BTreeMap, HashMap}, + fmt::Debug, + hash::Hash, + marker::PhantomData, +}; use crate::middlewares::versioned_flat_key_value::table_schema::VersionedKeyValueSchema; +use super::PendingError; + pub trait PendingKeyValueSchema { type Key: Eq + Hash + Clone + Ord; type CommitId: Debug + Eq + Hash + Copy; type Value: Clone; } -pub type Key = ::Key; -pub type Value = ::Value; -pub type CommitId = ::CommitId; +type Key = ::Key; +type Value = ::Value; +type CommitId = ::CommitId; + +pub struct RecoverRecord { + pub value: Option, + pub last_commit_id: Option, +} + +pub struct ApplyRecord { + pub value: Option, + pub commit_id: S::CommitId, +} + +pub type RecoverMap = BTreeMap, RecoverRecord>; +pub type KeyValueMap = BTreeMap, Option>>; +pub type ApplyMap = BTreeMap, ApplyRecord>; -pub type Modifications = Vec<(Key, Option>, Option>)>; -pub type Commits = HashMap, (CommitId, Option>)>; +pub type CommitIdVec = Vec>; +pub type Result = std::result::Result>>; pub struct PendingKeyValueConfig { _marker: PhantomData<(T, CId)>, diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 3ac14b3..1a99b84 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -1,52 +1,83 @@ use std::collections::{BTreeMap, HashMap}; +use crate::middlewares::versioned_flat_key_value::pending_part::pending_schema::RecoverRecord; + use super::{ commit_tree::Tree, - pending_schema::{Commits, PendingKeyValueSchema}, + pending_schema::{ + ApplyMap, ApplyRecord, KeyValueMap, PendingKeyValueSchema, Result as PendResult, + }, PendingError, }; pub struct VersionedHashMap { parent_of_root: Option, - - #[allow(clippy::type_complexity)] - history: HashMap>>, tree: Tree, + history: HashMap>, + current: Option>, +} + +pub struct CurrentMap { + pub map: BTreeMap>, + pub commit_id: S::CommitId, +} + +impl CurrentMap { + pub fn new(commit_id: S::CommitId) -> Self { + Self { + map: BTreeMap::new(), + commit_id, + } + } + + fn rollback( + &mut self, + rollbacks: BTreeMap>, + history: &HashMap>, + ) { + for (key, to_commit_id) in rollbacks.into_iter() { + match to_commit_id { + None => { + self.map.remove(&key); + } + Some(commit_id) => { + let value = history.get(&commit_id).unwrap().get(&key).unwrap().clone(); + self.map.insert(key, ApplyRecord { commit_id, value }); + } + } + } + } - #[allow(clippy::type_complexity)] - current: BTreeMap)>, - current_node: Option, + fn apply(&mut self, applys: ApplyMap) { + for (key, apply) in applys.into_iter() { + self.map.insert(key, apply); + } + } } impl VersionedHashMap { pub fn new(parent_of_root: Option) -> Self { VersionedHashMap { parent_of_root, - current: BTreeMap::new(), history: HashMap::new(), tree: Tree::new(), - current_node: None, + current: None, } } } impl VersionedHashMap { - #[allow(clippy::type_complexity)] pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> Result< - Vec<(S::CommitId, Option>>)>, - PendingError, - > { + ) -> PendResult>)>, S> { let (to_commit_rev, to_remove) = self.tree.change_root(commit_id)?; if to_commit_rev.is_empty() { assert!(to_remove.is_empty()); return Ok(Vec::new()); } self.parent_of_root = Some(to_commit_rev[0]); - self.current.clear(); - self.current_node = None; + self.current = None; for to_remove_one in to_remove.into_iter() { self.history.remove(&to_remove_one); } @@ -63,29 +94,27 @@ impl VersionedHashMap { &mut self, updates: BTreeMap>, commit_id: S::CommitId, - parent_commit_id: Option, - ) -> Result<(), PendingError> { + parent_commit_id: S::CommitId, + ) -> PendResult<(), S> { if self.history.contains_key(&commit_id) { return Err(PendingError::CommitIdAlreadyExists(commit_id)); } // let parent to be self.current - self.walk_to_node(parent_commit_id)?; - assert_eq!(parent_commit_id, self.current_node); + self.checkout_current(parent_commit_id)?; + let current = self.current.as_ref().unwrap(); // add node - let mut modifications = Vec::new(); - let mut inner_map = HashMap::new(); - for (key, value) in updates.into_iter() { - inner_map.insert(key.clone(), value.clone()); - let old_commit_id = if let Some((old_commit_id, _)) = - self.current.insert(key.clone(), (commit_id, value.clone())) - { - Some(old_commit_id) - } else { - None - }; - modifications.push((key, value, old_commit_id)); + let mut modifications = BTreeMap::new(); + for (key, value) in updates.iter() { + let last_commit_id = current.map.get(key).map(|s| s.commit_id); + modifications.insert( + key.clone(), + RecoverRecord { + value: value.clone(), + last_commit_id, + }, + ); } - self.history.insert(commit_id, inner_map); + self.history.insert(commit_id, updates); self.tree .add_node(commit_id, parent_commit_id, modifications)?; self.current_node = Some(commit_id); @@ -99,13 +128,12 @@ impl VersionedHashMap { &mut self, commit_id: S::CommitId, key: &S::Key, - ) -> Result>, PendingError> { + ) -> PendResult>, S> { // let queried node to be self.current - self.walk_to_node_unchecked(commit_id)?; + self.checkout_current(commit_id)?; assert_eq!(Some(commit_id), self.current_node); // query - let value = self.current.get(key).map(|(_, value)| value.clone()); - Ok(value) + Ok(self.current.get(key).map(|c| c.value.clone())) } pub fn get_parent_of_root(&self) -> Option { @@ -114,51 +142,24 @@ impl VersionedHashMap { } impl VersionedHashMap { - fn walk_to_node( - &mut self, - target_commit_id: Option, - ) -> Result<(), PendingError> { - if let Some(target_commit_id) = target_commit_id { - self.walk_to_node_unchecked(target_commit_id)?; - } else { - self.current = BTreeMap::new(); - self.current_node = None; - } - Ok(()) - } - fn walk_to_node_unchecked( - &mut self, - target_commit_id: S::CommitId, - ) -> Result<(), PendingError> { - let (rollbacks, commits_rev) = if let Some(current_commit_id) = self.current_node { - self.tree.lca(current_commit_id, target_commit_id)? + fn checkout_current(&mut self, target_commit_id: S::CommitId) -> PendResult<(), S> { + if let Some(ref mut current) = self.current { + let (rollbacks, applys) = self + .tree + .collect_rollback_and_apply_ops(current.commit_id, target_commit_id)?; + current.rollback(rollbacks, &self.history); + current.apply(applys); + current.commit_id = target_commit_id; } else { - (HashMap::new(), self.tree.find_path(target_commit_id)?) + let mut current = CurrentMap::::new(target_commit_id); + let applys = self.tree.get_apply_map_from_root(target_commit_id)?; + current.apply(applys); + self.current = Some(current) }; - self.rollback_without_node_update(rollbacks); - self.commit_without_node_update(commits_rev); - self.current_node = Some(target_commit_id); - Ok(()) - } - fn commit_without_node_update(&mut self, commits_rev: Commits) { - for (key, (commit_id, value)) in commits_rev.into_iter() { - self.current.insert(key, (commit_id, value)); - } - } + assert_eq!(self.current.unwrap().commit_id, target_commit_id); - fn rollback_without_node_update(&mut self, rollbacks: HashMap>) { - for (key, old_commit_id) in rollbacks.into_iter() { - match old_commit_id { - None => { - self.current.remove(&key); - } - Some(old_commit_id) => { - let value = self.history.get(&old_commit_id).unwrap().get(&key).unwrap(); - self.current.insert(key, (old_commit_id, value.clone())); - } - } - } + Ok(()) } } @@ -223,7 +224,15 @@ mod tests { } let updates_none = updates .iter() - .map(|(key, value)| (key.clone(), value.clone(), None)) + .map(|(key, value)| { + ( + *key, + RecoverRecord { + value: *value, + last_commit_id: None, + }, + ) + }) .collect(); forward_only_tree @@ -252,8 +261,10 @@ mod tests { for ikey in 0..10 { let key: u64 = ikey; let result = versioned_hash_map.query(commit_id, &key).unwrap(); - let current = forward_only_tree.find_path(commit_id).unwrap(); - let answer = current.get(&key).and_then(|(_, value)| Some(value.clone())); + let current = forward_only_tree + .get_apply_map_from_root(commit_id) + .unwrap(); + let answer = current.get(&key).map(|a| a.value.clone()); assert_eq!(result, answer); } } @@ -264,13 +275,15 @@ mod tests { let mut forward_only_tree = Tree::::new(); let mut versioned_hash_map = VersionedHashMap::::new(None); - forward_only_tree.add_node(0, None, Vec::new()).unwrap(); + forward_only_tree + .add_node(0, None, BTreeMap::new()) + .unwrap(); versioned_hash_map .add_node(BTreeMap::new(), 0, None) .unwrap(); assert_eq!( - forward_only_tree.add_node(1, None, Vec::new()), + forward_only_tree.add_node(1, None, BTreeMap::new()), Err(PendingError::MultipleRootsNotAllowed) ); assert_eq!( @@ -285,7 +298,7 @@ mod tests { let mut versioned_hash_map = VersionedHashMap::::new(None); assert_eq!( - forward_only_tree.add_node(1, Some(0), Vec::new()), + forward_only_tree.add_node(1, Some(0), BTreeMap::new()), Err(PendingError::CommitIDNotFound(0)) ); assert_eq!( @@ -299,13 +312,15 @@ mod tests { let mut forward_only_tree = Tree::::new(); let mut versioned_hash_map = VersionedHashMap::::new(None); - forward_only_tree.add_node(0, None, Vec::new()).unwrap(); + forward_only_tree + .add_node(0, None, BTreeMap::new()) + .unwrap(); versioned_hash_map .add_node(BTreeMap::new(), 0, None) .unwrap(); assert_eq!( - forward_only_tree.add_node(0, Some(0), Vec::new()), + forward_only_tree.add_node(0, Some(0), BTreeMap::new()), Err(PendingError::CommitIdAlreadyExists(0)) ); assert_eq!( From 07e7fe2e519c12687b580bff5501208d449aaa69 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 13 Sep 2024 17:06:56 +0800 Subject: [PATCH 41/79] separate add_root & add_non_root_node --- .../versioned_flat_key_value/mod.rs | 2 +- .../pending_part/commit_tree.rs | 63 ++++++++---- .../pending_part/error.rs | 2 + .../pending_part/pending_schema.rs | 9 +- .../pending_part/versioned_hash_map.rs | 97 ++++++++++++++----- 5 files changed, 126 insertions(+), 47 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 002e082..2454e91 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -2,7 +2,7 @@ mod pending_part; mod serde; mod table_schema; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; pub use pending_part::PendingError; diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 8366e75..7e1f144 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -76,35 +76,54 @@ impl Tree { } impl Tree { - pub fn add_node( + pub fn add_root( &mut self, commit_id: S::CommitId, - parent_commit_id: Option, + modifications: RecoverMap, + ) -> PendResult<(), S> { + if !self.index_map.is_empty() { + return Err(PendingError::MultipleRootsNotAllowed); + } + // PendingError::CommitIdAlreadyExists(_) cannot happend because self.index_map is empty + + // new root + let root = TreeNode::new_root(commit_id, modifications); + + // add root to tree + let slab_index = self.nodes.insert(root); + self.index_map.insert(commit_id, slab_index); + + Ok(()) + } + + pub fn add_non_root_node( + &mut self, + commit_id: S::CommitId, + parent_commit_id: S::CommitId, modifications: RecoverMap, ) -> PendResult<(), S> { // return error if Some(parent_commit_id) but parent_commit_id does not exist - let (parent_slab_index, parent_height) = if let Some(parent_commit_id) = parent_commit_id { - let p_slab_index = self.get_slab_index_by_commit_id(parent_commit_id)?; - let p_height = self.get_node_by_slab_index(p_slab_index).height; - (Some(p_slab_index), p_height) - } else { - // return error if want to add root but there has been a root - if self.has_root() { - return Err(PendingError::MultipleRootsNotAllowed); - } - (None, 0) - }; + let parent_slab_index = self.get_slab_index_by_commit_id(parent_commit_id)?; + let parent_height = self.get_node_by_slab_index(parent_slab_index).height; + // return error if commit_id exists if self.index_map.contains_key(&commit_id) { return Err(PendingError::CommitIdAlreadyExists(commit_id)); } - let node = TreeNode::new(commit_id, parent_slab_index, parent_height, modifications); + // new node + let node = TreeNode::new( + commit_id, + Some(parent_slab_index), + parent_height, + modifications, + ); + + // add node to tree let slab_index = self.nodes.insert(node); self.index_map.insert(commit_id, slab_index); - if let Some(parent_slab_index) = parent_slab_index { - self.nodes[parent_slab_index].children.insert(slab_index); - } + self.nodes[parent_slab_index].children.insert(slab_index); + Ok(()) } } @@ -192,6 +211,7 @@ impl Tree { } // correctness based on single root + #[allow(clippy::type_complexity)] pub(super) fn collect_rollback_and_apply_ops( &self, current_commit_id: S::CommitId, @@ -233,6 +253,15 @@ impl Tree { } impl TreeNode { + pub fn new_root(commit_id: S::CommitId, modifications: RecoverMap) -> Self { + Self { + height: 0, + commit_id, + parent: None, + children: BTreeSet::new(), + modifications, + } + } pub fn new( commit_id: S::CommitId, parent: Option, diff --git a/src/middlewares/versioned_flat_key_value/pending_part/error.rs b/src/middlewares/versioned_flat_key_value/pending_part/error.rs index 295b891..e7f6bca 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/error.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/error.rs @@ -10,4 +10,6 @@ pub enum PendingError { MultipleRootsNotAllowed, #[error("commit id already exists")] CommitIdAlreadyExists(CommitId), + #[error("non_root node should have parent")] + NonRootNodeHasNoParentError, } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs index 0ba0ba7..121a780 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/pending_schema.rs @@ -1,9 +1,4 @@ -use std::{ - collections::{BTreeMap, HashMap}, - fmt::Debug, - hash::Hash, - marker::PhantomData, -}; +use std::{collections::BTreeMap, fmt::Debug, hash::Hash, marker::PhantomData}; use crate::middlewares::versioned_flat_key_value::table_schema::VersionedKeyValueSchema; @@ -29,8 +24,8 @@ pub struct ApplyRecord { pub commit_id: S::CommitId, } -pub type RecoverMap = BTreeMap, RecoverRecord>; pub type KeyValueMap = BTreeMap, Option>>; +pub type RecoverMap = BTreeMap, RecoverRecord>; pub type ApplyMap = BTreeMap, ApplyRecord>; pub type CommitIdVec = Vec>; diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 1a99b84..1cff8e8 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -67,6 +67,7 @@ impl VersionedHashMap { } impl VersionedHashMap { + #[allow(clippy::type_complexity)] pub fn change_root( &mut self, commit_id: S::CommitId, @@ -92,17 +93,61 @@ impl VersionedHashMap { impl VersionedHashMap { pub fn add_node( &mut self, - updates: BTreeMap>, + updates: KeyValueMap, commit_id: S::CommitId, - parent_commit_id: S::CommitId, + parent_commit_id: Option, ) -> PendResult<(), S> { if self.history.contains_key(&commit_id) { return Err(PendingError::CommitIdAlreadyExists(commit_id)); } + + if self.get_parent_of_root() == parent_commit_id { + self.add_root(updates, commit_id) + } else if let Some(parent_commit_id) = parent_commit_id { + self.add_non_root_node(updates, commit_id, parent_commit_id) + } else { + Err(PendingError::NonRootNodeHasNoParentError) + } + } + + fn add_root(&mut self, updates: KeyValueMap, commit_id: S::CommitId) -> PendResult<(), S> { + if self.current.is_some() || !self.history.is_empty() { + return Err(PendingError::MultipleRootsNotAllowed); + } + + // add root to tree + let modifications = updates + .iter() + .map(|(k, v)| { + ( + k.clone(), + RecoverRecord:: { + value: v.clone(), + last_commit_id: None, + }, + ) + }) + .collect(); + self.tree.add_root(commit_id, modifications)?; + + // add root to history + self.history.insert(commit_id, updates); + + Ok(()) + } + + fn add_non_root_node( + &mut self, + updates: KeyValueMap, + commit_id: S::CommitId, + parent_commit_id: S::CommitId, + ) -> PendResult<(), S> { // let parent to be self.current + // this step is necessary for computing modifications' last_commit_id self.checkout_current(parent_commit_id)?; + + // add node to tree let current = self.current.as_ref().unwrap(); - // add node let mut modifications = BTreeMap::new(); for (key, value) in updates.iter() { let last_commit_id = current.map.get(key).map(|s| s.commit_id); @@ -114,10 +159,12 @@ impl VersionedHashMap { }, ); } - self.history.insert(commit_id, updates); self.tree - .add_node(commit_id, parent_commit_id, modifications)?; - self.current_node = Some(commit_id); + .add_non_root_node(commit_id, parent_commit_id, modifications)?; + + // add node to history + self.history.insert(commit_id, updates); + Ok(()) } @@ -129,11 +176,17 @@ impl VersionedHashMap { commit_id: S::CommitId, key: &S::Key, ) -> PendResult>, S> { - // let queried node to be self.current + // let query node to be self.current self.checkout_current(commit_id)?; - assert_eq!(Some(commit_id), self.current_node); + // query - Ok(self.current.get(key).map(|c| c.value.clone())) + Ok(self + .current + .as_ref() + .unwrap() + .map + .get(key) + .map(|c| c.value.clone())) } pub fn get_parent_of_root(&self) -> Option { @@ -157,7 +210,7 @@ impl VersionedHashMap { self.current = Some(current) }; - assert_eq!(self.current.unwrap().commit_id, target_commit_id); + assert_eq!(self.current.as_ref().unwrap().commit_id, target_commit_id); Ok(()) } @@ -235,9 +288,13 @@ mod tests { }) .collect(); - forward_only_tree - .add_node(i, parent_commit_id, updates_none) - .unwrap(); + if let Some(parent_commit_id) = parent_commit_id { + forward_only_tree + .add_non_root_node(i, parent_commit_id, updates_none) + .unwrap(); + } else { + forward_only_tree.add_root(i, updates_none).unwrap(); + } versioned_hash_map .add_node(updates, i, parent_commit_id) .unwrap(); @@ -275,15 +332,13 @@ mod tests { let mut forward_only_tree = Tree::::new(); let mut versioned_hash_map = VersionedHashMap::::new(None); - forward_only_tree - .add_node(0, None, BTreeMap::new()) - .unwrap(); + forward_only_tree.add_root(0, BTreeMap::new()).unwrap(); versioned_hash_map .add_node(BTreeMap::new(), 0, None) .unwrap(); assert_eq!( - forward_only_tree.add_node(1, None, BTreeMap::new()), + forward_only_tree.add_root(1, BTreeMap::new()), Err(PendingError::MultipleRootsNotAllowed) ); assert_eq!( @@ -298,7 +353,7 @@ mod tests { let mut versioned_hash_map = VersionedHashMap::::new(None); assert_eq!( - forward_only_tree.add_node(1, Some(0), BTreeMap::new()), + forward_only_tree.add_non_root_node(1, 0, BTreeMap::new()), Err(PendingError::CommitIDNotFound(0)) ); assert_eq!( @@ -312,15 +367,13 @@ mod tests { let mut forward_only_tree = Tree::::new(); let mut versioned_hash_map = VersionedHashMap::::new(None); - forward_only_tree - .add_node(0, None, BTreeMap::new()) - .unwrap(); + forward_only_tree.add_root(0, BTreeMap::new()).unwrap(); versioned_hash_map .add_node(BTreeMap::new(), 0, None) .unwrap(); assert_eq!( - forward_only_tree.add_node(0, Some(0), BTreeMap::new()), + forward_only_tree.add_non_root_node(0, 0, BTreeMap::new()), Err(PendingError::CommitIdAlreadyExists(0)) ); assert_eq!( From f280dad4cb2d3bc1e214d30f04feabc788b14f05 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 13 Sep 2024 17:39:54 +0800 Subject: [PATCH 42/79] remove unused fn --- .../pending_part/commit_tree.rs | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 7e1f144..9a7faa7 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -16,7 +16,7 @@ pub struct TreeNode { // todo: test lazy height // height will not be changed even when root is changed - // height is only used for lca + // height is only used for lca in Tree.collect_rollback_and_apply_ops() height: usize, commit_id: S::CommitId, @@ -64,11 +64,6 @@ impl Tree { !self.index_map.is_empty() } - pub fn get_parent_commit_id(&self, node: &TreeNode) -> Option { - node.parent - .map(|p_slab_index| self.nodes[p_slab_index].commit_id) - } - fn get_parent_node(&self, node: &TreeNode) -> Option<&TreeNode> { node.parent .map(|p_slab_index| self.get_node_by_slab_index(p_slab_index)) @@ -81,10 +76,11 @@ impl Tree { commit_id: S::CommitId, modifications: RecoverMap, ) -> PendResult<(), S> { - if !self.index_map.is_empty() { + // return error if there is root + if self.has_root() { return Err(PendingError::MultipleRootsNotAllowed); } - // PendingError::CommitIdAlreadyExists(_) cannot happend because self.index_map is empty + // PendingError::CommitIdAlreadyExists(_) cannot happend because no root indicates no node // new root let root = TreeNode::new_root(commit_id, modifications); @@ -102,20 +98,19 @@ impl Tree { parent_commit_id: S::CommitId, modifications: RecoverMap, ) -> PendResult<(), S> { - // return error if Some(parent_commit_id) but parent_commit_id does not exist + // return error if parent_commit_id does not exist let parent_slab_index = self.get_slab_index_by_commit_id(parent_commit_id)?; - let parent_height = self.get_node_by_slab_index(parent_slab_index).height; - + // return error if commit_id exists - if self.index_map.contains_key(&commit_id) { + if self.contains_commit_id(&commit_id) { return Err(PendingError::CommitIdAlreadyExists(commit_id)); } // new node let node = TreeNode::new( commit_id, - Some(parent_slab_index), - parent_height, + parent_slab_index, + self.get_node_by_slab_index(parent_slab_index).height + 1, modifications, ); @@ -264,14 +259,14 @@ impl TreeNode { } pub fn new( commit_id: S::CommitId, - parent: Option, - parent_height: usize, + parent: SlabIndex, + height: usize, modifications: RecoverMap, ) -> Self { Self { - height: parent_height + 1, + height, commit_id, - parent, + parent: Some(parent), children: BTreeSet::new(), modifications, } From ae659e44a09c0a0ef256f8f8dfef781cc981d65d Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 13 Sep 2024 18:36:43 +0800 Subject: [PATCH 43/79] rename get_apply_map_from_root --- .../versioned_flat_key_value/pending_part/commit_tree.rs | 2 +- .../pending_part/versioned_hash_map.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 9a7faa7..d71f23a 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -191,7 +191,7 @@ impl Tree { } impl Tree { - pub fn get_apply_map_from_root( + pub fn get_apply_map_from_root_included( &self, target_commit_id: S::CommitId, ) -> PendResult, S> { diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 1cff8e8..7562140 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -205,7 +205,7 @@ impl VersionedHashMap { current.commit_id = target_commit_id; } else { let mut current = CurrentMap::::new(target_commit_id); - let applys = self.tree.get_apply_map_from_root(target_commit_id)?; + let applys = self.tree.get_apply_map_from_root_included(target_commit_id)?; current.apply(applys); self.current = Some(current) }; @@ -319,7 +319,7 @@ mod tests { let key: u64 = ikey; let result = versioned_hash_map.query(commit_id, &key).unwrap(); let current = forward_only_tree - .get_apply_map_from_root(commit_id) + .get_apply_map_from_root_included(commit_id) .unwrap(); let answer = current.get(&key).map(|a| a.value.clone()); assert_eq!(result, answer); From 39b2672c19c7bed914d8085d51cc13c2d1832ae3 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 13 Sep 2024 18:50:04 +0800 Subject: [PATCH 44/79] cargo fmt --- .../versioned_flat_key_value/pending_part/commit_tree.rs | 3 ++- .../pending_part/versioned_hash_map.rs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index d71f23a..a1cd523 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -100,7 +100,7 @@ impl Tree { ) -> PendResult<(), S> { // return error if parent_commit_id does not exist let parent_slab_index = self.get_slab_index_by_commit_id(parent_commit_id)?; - + // return error if commit_id exists if self.contains_commit_id(&commit_id) { return Err(PendingError::CommitIdAlreadyExists(commit_id)); @@ -257,6 +257,7 @@ impl TreeNode { modifications, } } + pub fn new( commit_id: S::CommitId, parent: SlabIndex, diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 7562140..5782691 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -205,7 +205,9 @@ impl VersionedHashMap { current.commit_id = target_commit_id; } else { let mut current = CurrentMap::::new(target_commit_id); - let applys = self.tree.get_apply_map_from_root_included(target_commit_id)?; + let applys = self + .tree + .get_apply_map_from_root_included(target_commit_id)?; current.apply(applys); self.current = Some(current) }; From 70b560e1a12c1456151e8cd00150b554f6bda9b2 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 13 Sep 2024 18:52:51 +0800 Subject: [PATCH 45/79] remove unused func --- .../pending_part/commit_tree.rs | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index a1cd523..116183a 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -140,7 +140,7 @@ impl Tree { slab_indices } - fn find_path_nodes( + fn find_path( &self, target_slab_index: SlabIndex, ) -> (CommitIdVec, HashSet) { @@ -163,7 +163,7 @@ impl Tree { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; // (root)..=(new_root's parent) - let (to_commit_rev, to_commit_set) = self.find_path_nodes(slab_index); + let (to_commit_rev, to_commit_set) = self.find_path(slab_index); // subtree of new_root let to_maintain_vec = self.bfs_subtree(slab_index); @@ -248,7 +248,7 @@ impl Tree { } impl TreeNode { - pub fn new_root(commit_id: S::CommitId, modifications: RecoverMap) -> Self { + fn new_root(commit_id: S::CommitId, modifications: RecoverMap) -> Self { Self { height: 0, commit_id, @@ -258,7 +258,7 @@ impl TreeNode { } } - pub fn new( + fn new( commit_id: S::CommitId, parent: SlabIndex, height: usize, @@ -273,21 +273,13 @@ impl TreeNode { } } - pub fn get_commit_id(&self) -> S::CommitId { - self.commit_id - } - - pub fn get_modifications(&self) -> &RecoverMap { - &self.modifications - } - - pub fn export_rollback_data(&self, rollbacks: &mut BTreeMap>) { + fn export_rollback_data(&self, rollbacks: &mut BTreeMap>) { for (key, RecoverRecord { last_commit_id, .. }) in self.modifications.iter() { rollbacks.insert(key.clone(), *last_commit_id); } } - pub fn export_commit_data(&self, commits_rev: &mut ApplyMap) { + fn export_commit_data(&self, commits_rev: &mut ApplyMap) { let commit_id = self.commit_id; for (key, RecoverRecord { value, .. }) in self.modifications.iter() { commits_rev From bb464d92713dbf126ff75d97636dad9f713e86f3 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:35:22 +0800 Subject: [PATCH 46/79] add comments --- .../pending_part/commit_tree.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 116183a..cd02703 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -124,6 +124,7 @@ impl Tree { } impl Tree { + // including subroot fn bfs_subtree(&self, subroot_slab_index: SlabIndex) -> Vec { let mut slab_indices = vec![subroot_slab_index]; let mut head = 0; @@ -140,10 +141,8 @@ impl Tree { slab_indices } - fn find_path( - &self, - target_slab_index: SlabIndex, - ) -> (CommitIdVec, HashSet) { + // excluding target + fn find_path(&self, target_slab_index: SlabIndex) -> (CommitIdVec, HashSet) { let mut target_node = self.get_node_by_slab_index(target_slab_index); let mut path = Vec::new(); let mut set = HashSet::new(); @@ -162,14 +161,14 @@ impl Tree { ) -> PendResult<(CommitIdVec, CommitIdVec), S> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; - // (root)..=(new_root's parent) + // (root..=new_root's parent).rev() let (to_commit_rev, to_commit_set) = self.find_path(slab_index); // subtree of new_root let to_maintain_vec = self.bfs_subtree(slab_index); let to_maintain = BTreeSet::from_iter(to_maintain_vec); - // tree - subtree of new_root - (root)..-(new_root's parent) + // tree - subtree of new_root - root..=new_root's parent let mut to_remove_indices = Vec::new(); for (idx, _) in self.nodes.iter() { if !to_maintain.contains(&idx) && !to_commit_set.contains(&idx) { @@ -178,9 +177,9 @@ impl Tree { } let mut to_remove = Vec::new(); for idx in to_remove_indices.into_iter() { - let to_remove_node = self.nodes.remove(idx); - self.index_map.remove(&to_remove_node.commit_id); - to_remove.push(to_remove_node.commit_id); + let to_remove_commit_id = self.nodes.remove(idx).commit_id; + self.index_map.remove(&to_remove_commit_id); + to_remove.push(to_remove_commit_id); } // set new_root's parent as None From bd5984ec1a570dd6e5bebddf6325ecb8dff89f7e Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:04:44 +0800 Subject: [PATCH 47/79] fix bug in change_root --- .../versioned_flat_key_value/pending_part/commit_tree.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index cd02703..6331608 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -168,7 +168,7 @@ impl Tree { let to_maintain_vec = self.bfs_subtree(slab_index); let to_maintain = BTreeSet::from_iter(to_maintain_vec); - // tree - subtree of new_root - root..=new_root's parent + // remove: tree - subtree of new_root - root..=new_root's parent let mut to_remove_indices = Vec::new(); for (idx, _) in self.nodes.iter() { if !to_maintain.contains(&idx) && !to_commit_set.contains(&idx) { @@ -185,6 +185,12 @@ impl Tree { // set new_root's parent as None self.nodes[slab_index].parent = None; + // remove: root..=new_root's parent + for idx in to_commit_set.into_iter() { + self.index_map.remove(&self.nodes.remove(idx).commit_id); + } + + // ((root..=new_root's parent).rev(), tree - subtree of new_root - root..=new_root's parent) Ok((to_commit_rev, to_remove)) } } From 40a6b37ef67a2a070a9064bb16f4137ce083dc07 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:09:31 +0800 Subject: [PATCH 48/79] remove unnecessary pub --- .../pending_part/mod.rs | 2 +- .../pending_part/versioned_hash_map.rs | 20 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs index e76a045..2e154a3 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs @@ -1,4 +1,4 @@ -pub mod commit_tree; +mod commit_tree; pub mod error; pub mod pending_schema; pub mod versioned_hash_map; diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 5782691..f47c53c 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -1,11 +1,10 @@ use std::collections::{BTreeMap, HashMap}; -use crate::middlewares::versioned_flat_key_value::pending_part::pending_schema::RecoverRecord; - use super::{ commit_tree::Tree, pending_schema::{ - ApplyMap, ApplyRecord, KeyValueMap, PendingKeyValueSchema, Result as PendResult, + ApplyMap, ApplyRecord, KeyValueMap, PendingKeyValueSchema, RecoverRecord, + Result as PendResult, }, PendingError, }; @@ -17,13 +16,13 @@ pub struct VersionedHashMap { current: Option>, } -pub struct CurrentMap { - pub map: BTreeMap>, - pub commit_id: S::CommitId, +struct CurrentMap { + map: BTreeMap>, + commit_id: S::CommitId, } impl CurrentMap { - pub fn new(commit_id: S::CommitId) -> Self { + fn new(commit_id: S::CommitId) -> Self { Self { map: BTreeMap::new(), commit_id, @@ -68,10 +67,13 @@ impl VersionedHashMap { impl VersionedHashMap { #[allow(clippy::type_complexity)] + // root..=new_root's parent: (commit_id, key_value_map) pub fn change_root( &mut self, commit_id: S::CommitId, ) -> PendResult>)>, S> { + // to_commit_rev: (root..=new_root's parent).rev() + // to_remove: tree - subtree of new_root - root..=new_root's parent let (to_commit_rev, to_remove) = self.tree.change_root(commit_id)?; if to_commit_rev.is_empty() { assert!(to_remove.is_empty()); @@ -320,10 +322,10 @@ mod tests { for ikey in 0..10 { let key: u64 = ikey; let result = versioned_hash_map.query(commit_id, &key).unwrap(); - let current = forward_only_tree + let apply_map = forward_only_tree .get_apply_map_from_root_included(commit_id) .unwrap(); - let answer = current.get(&key).map(|a| a.value.clone()); + let answer = apply_map.get(&key).map(|a| a.value.clone()); assert_eq!(result, answer); } } From 9a6220ef0d6ab89af36f34b795aa7a28d0c0a183 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 23 Sep 2024 11:54:22 +0800 Subject: [PATCH 49/79] impl confirmed_pending_to_history --- src/middlewares/key_value_store_bulks.rs | 5 ++-- .../versioned_flat_key_value/mod.rs | 23 ++++++++++++++++++- .../pending_part/versioned_hash_map.rs | 6 ++--- src/traits.rs | 2 +- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/middlewares/key_value_store_bulks.rs b/src/middlewares/key_value_store_bulks.rs index 8b456f2..47eadb1 100644 --- a/src/middlewares/key_value_store_bulks.rs +++ b/src/middlewares/key_value_store_bulks.rs @@ -30,10 +30,11 @@ where fn commit( &self, commit: C, - bulk: impl Iterator, + bulk: impl Iterator)>, write_schema: &impl WriteSchemaTrait, ) -> Result<()> { - let table_op = bulk.map(|(k, v)| (Cow::Owned(ChangeKey(commit, k)), Some(Cow::Owned(v)))); + let table_op = + bulk.map(|(k, v)| (Cow::Owned(ChangeKey(commit, k)), v.map(|x| Cow::Owned(x)))); write_schema.write_batch::(table_op); Ok(()) } diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 2454e91..ae05e2c 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -12,7 +12,7 @@ use pending_part::VersionedHashMap; use super::ChangeKey; use super::CommitIDSchema; -use crate::backends::TableReader; +use crate::backends::{TableReader, WriteSchemaTrait}; use crate::errors::Result; use crate::middlewares::{CommitID, HistoryNumber, KeyValueStoreBulks}; use crate::traits::KeyValueStoreBulksTrait; @@ -98,3 +98,24 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { .get_versioned_key(&found_version_number, key) } } + +impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { + fn confirmed_pending_to_history( + &mut self, + new_root_commit_id: CommitID, + write_schema: &impl WriteSchemaTrait, + ) -> Result<()> { + // old root..=new root's parent + let confirmed_ids_maps = self.pending_part.change_root(new_root_commit_id)?; + for (confirmed_commit_id, updates) in confirmed_ids_maps.into_iter() { + // allocate a history_number to confired_commit_id + let history_number = 0; //todo!(); + // self.commit_id_table + // .set(confirmed_commit_id, history_number); + self.change_history_table + .commit(history_number, updates.into_iter(), write_schema)?; + // self.history_index_table; + } + Ok(()) + } +} diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index f47c53c..3cc790f 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -71,7 +71,7 @@ impl VersionedHashMap { pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> PendResult>)>, S> { + ) -> PendResult)>, S> { // to_commit_rev: (root..=new_root's parent).rev() // to_remove: tree - subtree of new_root - root..=new_root's parent let (to_commit_rev, to_remove) = self.tree.change_root(commit_id)?; @@ -82,11 +82,11 @@ impl VersionedHashMap { self.parent_of_root = Some(to_commit_rev[0]); self.current = None; for to_remove_one in to_remove.into_iter() { - self.history.remove(&to_remove_one); + self.history.remove(&to_remove_one).unwrap(); } let mut to_commit = Vec::new(); for to_commit_one in to_commit_rev.into_iter().rev() { - to_commit.push((to_commit_one, self.history.remove(&to_commit_one))); + to_commit.push((to_commit_one, self.history.remove(&to_commit_one).unwrap())); } Ok(to_commit) } diff --git a/src/traits.rs b/src/traits.rs index b5f81fc..0919113 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -43,7 +43,7 @@ pub trait KeyValueStoreBulksTrait { fn commit( &self, commit: C, - bulk: impl Iterator, + bulk: impl Iterator)>, write_schema: &impl WriteSchemaTrait, ) -> Result<()>; From 7027bbbc6fadae260f74af0147e2b68e7526a3ac Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:32:54 +0800 Subject: [PATCH 50/79] use height as history number --- src/middlewares/versioned_flat_key_value/mod.rs | 17 +++++++++++------ .../pending_part/commit_tree.rs | 8 ++++++-- .../pending_part/versioned_hash_map.rs | 9 +++++---- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index ae05e2c..9414f02 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -106,14 +106,19 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { write_schema: &impl WriteSchemaTrait, ) -> Result<()> { // old root..=new root's parent - let confirmed_ids_maps = self.pending_part.change_root(new_root_commit_id)?; - for (confirmed_commit_id, updates) in confirmed_ids_maps.into_iter() { - // allocate a history_number to confired_commit_id - let history_number = 0; //todo!(); + let (start_history_number, confirmed_ids_maps) = + self.pending_part.change_root(new_root_commit_id)?; + for (delta_history_number, (confirmed_commit_id, updates)) in + confirmed_ids_maps.into_iter().enumerate() + { + let history_number = start_history_number + delta_history_number; // self.commit_id_table // .set(confirmed_commit_id, history_number); - self.change_history_table - .commit(history_number, updates.into_iter(), write_schema)?; + self.change_history_table.commit( + history_number as u64, + updates.into_iter(), + write_schema, + )?; // self.history_index_table; } Ok(()) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 6331608..d33b2b0 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -158,7 +158,7 @@ impl Tree { pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> PendResult<(CommitIdVec, CommitIdVec), S> { + ) -> PendResult<(usize, CommitIdVec, CommitIdVec), S> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; // (root..=new_root's parent).rev() @@ -191,7 +191,11 @@ impl Tree { } // ((root..=new_root's parent).rev(), tree - subtree of new_root - root..=new_root's parent) - Ok((to_commit_rev, to_remove)) + Ok(( + self.nodes[slab_index].height - to_commit_rev.len(), + to_commit_rev, + to_remove, + )) } } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 3cc790f..eae124e 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -71,13 +71,14 @@ impl VersionedHashMap { pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> PendResult)>, S> { + ) -> PendResult<(usize, Vec<(S::CommitId, KeyValueMap)>), S> { // to_commit_rev: (root..=new_root's parent).rev() // to_remove: tree - subtree of new_root - root..=new_root's parent - let (to_commit_rev, to_remove) = self.tree.change_root(commit_id)?; + let (start_height_to_commit, to_commit_rev, to_remove) = + self.tree.change_root(commit_id)?; if to_commit_rev.is_empty() { assert!(to_remove.is_empty()); - return Ok(Vec::new()); + return Ok((0, Vec::new())); } self.parent_of_root = Some(to_commit_rev[0]); self.current = None; @@ -88,7 +89,7 @@ impl VersionedHashMap { for to_commit_one in to_commit_rev.into_iter().rev() { to_commit.push((to_commit_one, self.history.remove(&to_commit_one).unwrap())); } - Ok(to_commit) + Ok((start_height_to_commit, to_commit)) } } From 37300426e4dbfd2afdb5d0d3412bc89925bf6da2 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:41:44 +0800 Subject: [PATCH 51/79] index not use bitmap, !height as history_number --- src/middlewares/versioned_flat_key_value/mod.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 9414f02..1201476 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -77,6 +77,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { let range_query_key = HistoryIndexKey(key.clone(), target_history_number); + // history_number should be decreasing, use !height let found_version_number = match self.history_index_table.iter(&range_query_key)?.next() { None => { return Ok(None); @@ -89,8 +90,8 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { } Some(Ok((k, indices))) => { let HistoryIndexKey(_, history_number) = k.as_ref(); - let offset = target_history_number - history_number; - indices.as_ref().last(offset) + // let offset = target_history_number - history_number; + indices.as_ref().last(*history_number) } }; @@ -106,12 +107,13 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { write_schema: &impl WriteSchemaTrait, ) -> Result<()> { // old root..=new root's parent - let (start_history_number, confirmed_ids_maps) = + let (start_height, confirmed_ids_maps) = self.pending_part.change_root(new_root_commit_id)?; - for (delta_history_number, (confirmed_commit_id, updates)) in + for (delta_height, (confirmed_commit_id, updates)) in confirmed_ids_maps.into_iter().enumerate() { - let history_number = start_history_number + delta_history_number; + let height = (start_height + delta_height) as u64; + let history_number = !height; // self.commit_id_table // .set(confirmed_commit_id, history_number); self.change_history_table.commit( From 659585faa10048cb826623bc9c696d3b3f36a2cf Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:55:18 +0800 Subject: [PATCH 52/79] complete confirmed_pending_to_history --- .../versioned_flat_key_value/mod.rs | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 1201476..ddfbea9 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -2,6 +2,7 @@ mod pending_part; mod serde; mod table_schema; +use std::borrow::Cow; use std::collections::BTreeMap; pub use pending_part::PendingError; @@ -109,19 +110,29 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { // old root..=new root's parent let (start_height, confirmed_ids_maps) = self.pending_part.change_root(new_root_commit_id)?; + for (delta_height, (confirmed_commit_id, updates)) in confirmed_ids_maps.into_iter().enumerate() { let height = (start_height + delta_height) as u64; let history_number = !height; - // self.commit_id_table - // .set(confirmed_commit_id, history_number); - self.change_history_table.commit( - history_number as u64, - updates.into_iter(), - write_schema, - )?; - // self.history_index_table; + + let commit_id_table_op = ( + Cow::Owned(confirmed_commit_id), + Some(Cow::Owned(history_number)), + ); + write_schema.write::(commit_id_table_op); + + let history_index_table_op = updates.keys().map(|key| { + ( + Cow::Owned(HistoryIndexKey(key.clone(), history_number)), + Some(Cow::Owned(HistoryIndices)), + ) + }); + write_schema.write_batch::>(history_index_table_op); + + self.change_history_table + .commit(history_number, updates.into_iter(), write_schema)?; } Ok(()) } From c56e6ccbc5bb1379e5c1b2e0d0662d6c26c76749 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:59:04 +0800 Subject: [PATCH 53/79] add assertion --- src/middlewares/versioned_flat_key_value/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index ddfbea9..92cbe63 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -117,6 +117,8 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { let height = (start_height + delta_height) as u64; let history_number = !height; + assert!(self.commit_id_table.get(&confirmed_commit_id)?.is_none()); + let commit_id_table_op = ( Cow::Owned(confirmed_commit_id), Some(Cow::Owned(history_number)), From c8aa8f73772c5da1d3905217956d6059ca97f033 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:15:03 +0800 Subject: [PATCH 54/79] rename variable --- src/middlewares/versioned_flat_key_value/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 92cbe63..910124e 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -125,17 +125,18 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { ); write_schema.write::(commit_id_table_op); - let history_index_table_op = updates.keys().map(|key| { + let history_indices_table_op = updates.keys().map(|key| { ( Cow::Owned(HistoryIndexKey(key.clone(), history_number)), Some(Cow::Owned(HistoryIndices)), ) }); - write_schema.write_batch::>(history_index_table_op); + write_schema.write_batch::>(history_indices_table_op); self.change_history_table .commit(history_number, updates.into_iter(), write_schema)?; } + Ok(()) } } From 972eccfa78e4d79d18dfd46a4ceb2e77fb79d525 Mon Sep 17 00:00:00 2001 From: Chenxing Li Date: Tue, 24 Sep 2024 17:13:14 +0800 Subject: [PATCH 55/79] update get_pending_part --- .../versioned_flat_key_value/mod.rs | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 910124e..9ccb538 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -45,19 +45,29 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { pub fn get_pending_part(&mut self, commit: CommitID, key: &T::Key) -> Result> { let res_value = self.pending_part.query(commit, key); let history_commit = match res_value { - Ok(Some(value)) => return Ok(value), - Ok(None) => self.pending_part.get_parent_of_root(), + Ok(Some(value)) => { + return Ok(value); + } + Ok(None) => if let Some(commit) = self.pending_part.get_parent_of_root() { + commit + } else { + return Ok(None); + }, Err(PendingError::CommitIDNotFound(target_commit)) => { assert_eq!(target_commit, commit); - Some(commit) + commit + } + Err(other_err) => { + return Err(StorageError::PendingError(other_err)); } - Err(other_err) => return Err(StorageError::PendingError(other_err)), }; - if let Some(history_commit) = history_commit { - self.get_historical_part(history_commit, key) + + let history_number = if let Some(value) = self.commit_id_table.get(&commit)? { + value.into_owned() } else { - Ok(None) - } + return Err(StorageError::CommitIDNotFound); + }; + self.get_historical_part(history_number, key) } pub fn add_to_pending_part( @@ -69,14 +79,8 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { Ok(self.pending_part.add_node(updates, commit, parent_commit)?) } - pub fn get_historical_part(&self, commit: CommitID, key: &T::Key) -> Result> { - let target_history_number = if let Some(value) = self.commit_id_table.get(&commit)? { - value.into_owned() - } else { - return Err(StorageError::CommitIDNotFound); - }; - - let range_query_key = HistoryIndexKey(key.clone(), target_history_number); + pub fn get_historical_part(&self, query_version_number: HistoryNumber, key: &T::Key) -> Result> { + let range_query_key = HistoryIndexKey(key.clone(), query_version_number); // history_number should be decreasing, use !height let found_version_number = match self.history_index_table.iter(&range_query_key)?.next() { @@ -136,7 +140,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { self.change_history_table .commit(history_number, updates.into_iter(), write_schema)?; } - + Ok(()) } } From 280e04e092ffa869403b87a6d1fed2d82c5a6a6f Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:16:46 +0800 Subject: [PATCH 56/79] remove redundant history in pending part --- .../pending_part/commit_tree.rs | 92 ++++++++++++------- .../pending_part/versioned_hash_map.rs | 82 ++++++----------- 2 files changed, 88 insertions(+), 86 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index d33b2b0..e51a189 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -1,9 +1,9 @@ -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use slab::Slab; use super::pending_schema::{ - ApplyMap, ApplyRecord, CommitIdVec, PendingKeyValueSchema, RecoverMap, RecoverRecord, + ApplyMap, ApplyRecord, KeyValueMap, PendingKeyValueSchema, RecoverMap, RecoverRecord, Result as PendResult, }; use super::PendingError; @@ -68,6 +68,15 @@ impl Tree { node.parent .map(|p_slab_index| self.get_node_by_slab_index(p_slab_index)) } + + fn get_by_commit_id( + &self, + commit_id: S::CommitId, + key: &S::Key, + ) -> PendResult>, S> { + let node = self.get_node_by_commit_id(commit_id)?; + Ok(node.modifications.get(key).map(|v| v.value.clone())) + } } impl Tree { @@ -142,60 +151,52 @@ impl Tree { } // excluding target - fn find_path(&self, target_slab_index: SlabIndex) -> (CommitIdVec, HashSet) { + fn find_path(&self, target_slab_index: SlabIndex) -> Vec<(S::CommitId, KeyValueMap)> { let mut target_node = self.get_node_by_slab_index(target_slab_index); - let mut path = Vec::new(); - let mut set = HashSet::new(); + let mut path = VecDeque::new(); while let Some(parent_slab_index) = target_node.parent { - set.insert(parent_slab_index); target_node = self.get_node_by_slab_index(parent_slab_index); - path.push(target_node.commit_id); + path.push_front((target_node.commit_id, target_node.get_updates())); } - (path, set) + path.into() } // todo: test + #[allow(clippy::type_complexity)] pub fn change_root( &mut self, commit_id: S::CommitId, - ) -> PendResult<(usize, CommitIdVec, CommitIdVec), S> { + ) -> PendResult<(usize, Vec<(S::CommitId, KeyValueMap)>), S> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; - // (root..=new_root's parent).rev() - let (to_commit_rev, to_commit_set) = self.find_path(slab_index); + // root..=new_root's parent + let to_commit = self.find_path(slab_index); + + // early return if new_root == root + if to_commit.is_empty() { + return Ok((self.nodes[slab_index].height - to_commit.len(), to_commit)); + } // subtree of new_root let to_maintain_vec = self.bfs_subtree(slab_index); let to_maintain = BTreeSet::from_iter(to_maintain_vec); - // remove: tree - subtree of new_root - root..=new_root's parent - let mut to_remove_indices = Vec::new(); + // remove: tree - subtree of new_root + let mut to_remove = Vec::new(); for (idx, _) in self.nodes.iter() { - if !to_maintain.contains(&idx) && !to_commit_set.contains(&idx) { - to_remove_indices.push(idx); + if !to_maintain.contains(&idx) { + to_remove.push(idx); } } - let mut to_remove = Vec::new(); - for idx in to_remove_indices.into_iter() { - let to_remove_commit_id = self.nodes.remove(idx).commit_id; - self.index_map.remove(&to_remove_commit_id); - to_remove.push(to_remove_commit_id); + for idx in to_remove { + self.index_map.remove(&self.nodes.remove(idx).commit_id); } // set new_root's parent as None self.nodes[slab_index].parent = None; - // remove: root..=new_root's parent - for idx in to_commit_set.into_iter() { - self.index_map.remove(&self.nodes.remove(idx).commit_id); - } - - // ((root..=new_root's parent).rev(), tree - subtree of new_root - root..=new_root's parent) - Ok(( - self.nodes[slab_index].height - to_commit_rev.len(), - to_commit_rev, - to_remove, - )) + // (height of new_root's parent, root..=new_root's parent) + Ok((self.nodes[slab_index].height - to_commit.len(), to_commit)) } } @@ -220,7 +221,7 @@ impl Tree { &self, current_commit_id: S::CommitId, target_commit_id: S::CommitId, - ) -> PendResult<(BTreeMap>, ApplyMap), S> { + ) -> PendResult<(BTreeMap>>, ApplyMap), S> { let mut current_node = self.get_node_by_commit_id(current_commit_id).unwrap(); let mut target_node = self.get_node_by_commit_id(target_commit_id)?; let mut rollbacks = BTreeMap::new(); @@ -249,10 +250,28 @@ impl Tree { } } } + + let rollbacks_with_value: BTreeMap<_, _> = rollbacks + .into_iter() + .map(|(k, old_cid_opt)| match old_cid_opt { + None => (k, None), + Some(rollback_cid) => { + let rollback_value = self.get_by_commit_id(rollback_cid, &k).unwrap().unwrap(); + ( + k, + Some(ApplyRecord:: { + value: rollback_value, + commit_id: rollback_cid, + }), + ) + } + }) + .collect(); + // rollbacks or commits_rev may be empty, // they contain current and target (if they are not lca), respectively, // but they do not contain lca - Ok((rollbacks, commits_rev)) + Ok((rollbacks_with_value, commits_rev)) } } @@ -282,6 +301,13 @@ impl TreeNode { } } + fn get_updates(&self) -> KeyValueMap { + self.modifications + .iter() + .map(|(k, RecoverRecord { value, .. })| (k.clone(), value.clone())) + .collect() + } + fn export_rollback_data(&self, rollbacks: &mut BTreeMap>) { for (key, RecoverRecord { last_commit_id, .. }) in self.modifications.iter() { rollbacks.insert(key.clone(), *last_commit_id); diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index eae124e..df5f23d 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use super::{ commit_tree::Tree, @@ -12,7 +12,6 @@ use super::{ pub struct VersionedHashMap { parent_of_root: Option, tree: Tree, - history: HashMap>, current: Option>, } @@ -29,19 +28,14 @@ impl CurrentMap { } } - fn rollback( - &mut self, - rollbacks: BTreeMap>, - history: &HashMap>, - ) { - for (key, to_commit_id) in rollbacks.into_iter() { - match to_commit_id { + fn rollback(&mut self, rollbacks: BTreeMap>>) { + for (key, to_rollback) in rollbacks.into_iter() { + match to_rollback { None => { self.map.remove(&key); } - Some(commit_id) => { - let value = history.get(&commit_id).unwrap().get(&key).unwrap().clone(); - self.map.insert(key, ApplyRecord { commit_id, value }); + Some(to_rollback_record) => { + self.map.insert(key, to_rollback_record); } } } @@ -58,7 +52,6 @@ impl VersionedHashMap { pub fn new(parent_of_root: Option) -> Self { VersionedHashMap { parent_of_root, - history: HashMap::new(), tree: Tree::new(), current: None, } @@ -72,38 +65,29 @@ impl VersionedHashMap { &mut self, commit_id: S::CommitId, ) -> PendResult<(usize, Vec<(S::CommitId, KeyValueMap)>), S> { - // to_commit_rev: (root..=new_root's parent).rev() - // to_remove: tree - subtree of new_root - root..=new_root's parent - let (start_height_to_commit, to_commit_rev, to_remove) = - self.tree.change_root(commit_id)?; - if to_commit_rev.is_empty() { - assert!(to_remove.is_empty()); - return Ok((0, Vec::new())); - } - self.parent_of_root = Some(to_commit_rev[0]); - self.current = None; - for to_remove_one in to_remove.into_iter() { - self.history.remove(&to_remove_one).unwrap(); - } - let mut to_commit = Vec::new(); - for to_commit_one in to_commit_rev.into_iter().rev() { - to_commit.push((to_commit_one, self.history.remove(&to_commit_one).unwrap())); + // to_commit: root..=new_root's parent + let (start_height_to_commit, to_commit) = self.tree.change_root(commit_id)?; + + if let Some(parent_of_new_root) = to_commit.last() { + self.parent_of_root = Some(parent_of_new_root.0); + self.current = None; } + Ok((start_height_to_commit, to_commit)) } } impl VersionedHashMap { + pub fn get_parent_of_root(&self) -> Option { + self.parent_of_root + } + pub fn add_node( &mut self, updates: KeyValueMap, commit_id: S::CommitId, parent_commit_id: Option, ) -> PendResult<(), S> { - if self.history.contains_key(&commit_id) { - return Err(PendingError::CommitIdAlreadyExists(commit_id)); - } - if self.get_parent_of_root() == parent_commit_id { self.add_root(updates, commit_id) } else if let Some(parent_commit_id) = parent_commit_id { @@ -114,18 +98,18 @@ impl VersionedHashMap { } fn add_root(&mut self, updates: KeyValueMap, commit_id: S::CommitId) -> PendResult<(), S> { - if self.current.is_some() || !self.history.is_empty() { + if self.current.is_some() { return Err(PendingError::MultipleRootsNotAllowed); } // add root to tree let modifications = updates - .iter() - .map(|(k, v)| { + .into_iter() + .map(|(key, value)| { ( - k.clone(), + key, RecoverRecord:: { - value: v.clone(), + value, last_commit_id: None, }, ) @@ -133,9 +117,6 @@ impl VersionedHashMap { .collect(); self.tree.add_root(commit_id, modifications)?; - // add root to history - self.history.insert(commit_id, updates); - Ok(()) } @@ -152,12 +133,12 @@ impl VersionedHashMap { // add node to tree let current = self.current.as_ref().unwrap(); let mut modifications = BTreeMap::new(); - for (key, value) in updates.iter() { - let last_commit_id = current.map.get(key).map(|s| s.commit_id); + for (key, value) in updates.into_iter() { + let last_commit_id = current.map.get(&key).map(|s| s.commit_id); modifications.insert( - key.clone(), + key, RecoverRecord { - value: value.clone(), + value, last_commit_id, }, ); @@ -165,12 +146,11 @@ impl VersionedHashMap { self.tree .add_non_root_node(commit_id, parent_commit_id, modifications)?; - // add node to history - self.history.insert(commit_id, updates); - Ok(()) } +} +impl VersionedHashMap { // None: pending_part not know // Some(None): pending_part know that this key has been deleted // Some(Some(value)): pending_part know this key's value @@ -191,10 +171,6 @@ impl VersionedHashMap { .get(key) .map(|c| c.value.clone())) } - - pub fn get_parent_of_root(&self) -> Option { - self.parent_of_root - } } impl VersionedHashMap { @@ -203,7 +179,7 @@ impl VersionedHashMap { let (rollbacks, applys) = self .tree .collect_rollback_and_apply_ops(current.commit_id, target_commit_id)?; - current.rollback(rollbacks, &self.history); + current.rollback(rollbacks); current.apply(applys); current.commit_id = target_commit_id; } else { From c0ce2e17cf5286f3cc193c28978a86ee347151fa Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:31:43 +0800 Subject: [PATCH 57/79] fix history_number as height & make height unique --- .../versioned_flat_key_value/mod.rs | 24 ++++++++++++------- .../pending_part/commit_tree.rs | 10 ++++---- .../pending_part/versioned_hash_map.rs | 20 ++++++++-------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 9ccb538..578a266 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -48,11 +48,13 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { Ok(Some(value)) => { return Ok(value); } - Ok(None) => if let Some(commit) = self.pending_part.get_parent_of_root() { - commit - } else { - return Ok(None); - }, + Ok(None) => { + if let Some(commit) = self.pending_part.get_parent_of_root() { + commit + } else { + return Ok(None); + } + } Err(PendingError::CommitIDNotFound(target_commit)) => { assert_eq!(target_commit, commit); commit @@ -62,7 +64,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { } }; - let history_number = if let Some(value) = self.commit_id_table.get(&commit)? { + let history_number = if let Some(value) = self.commit_id_table.get(&commit)? { value.into_owned() } else { return Err(StorageError::CommitIDNotFound); @@ -79,10 +81,14 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { Ok(self.pending_part.add_node(updates, commit, parent_commit)?) } - pub fn get_historical_part(&self, query_version_number: HistoryNumber, key: &T::Key) -> Result> { + pub fn get_historical_part( + &self, + query_version_number: HistoryNumber, + key: &T::Key, + ) -> Result> { let range_query_key = HistoryIndexKey(key.clone(), query_version_number); - // history_number should be decreasing, use !height + // history_number should be decreasing let found_version_number = match self.history_index_table.iter(&range_query_key)?.next() { None => { return Ok(None); @@ -119,7 +125,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { confirmed_ids_maps.into_iter().enumerate() { let height = (start_height + delta_height) as u64; - let history_number = !height; + let history_number = height; assert!(self.commit_id_table.get(&confirmed_commit_id)?.is_none()); diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index e51a189..be2249f 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -27,13 +27,15 @@ pub struct TreeNode { } pub struct Tree { + height_of_root: usize, nodes: Slab>, index_map: HashMap, } impl Tree { - pub fn new() -> Self { + pub fn new(height_of_root: usize) -> Self { Tree { + height_of_root, nodes: Slab::new(), index_map: HashMap::new(), } @@ -92,7 +94,7 @@ impl Tree { // PendingError::CommitIdAlreadyExists(_) cannot happend because no root indicates no node // new root - let root = TreeNode::new_root(commit_id, modifications); + let root = TreeNode::new_root(commit_id, modifications, self.height_of_root); // add root to tree let slab_index = self.nodes.insert(root); @@ -276,9 +278,9 @@ impl Tree { } impl TreeNode { - fn new_root(commit_id: S::CommitId, modifications: RecoverMap) -> Self { + fn new_root(commit_id: S::CommitId, modifications: RecoverMap, height: usize) -> Self { Self { - height: 0, + height, commit_id, parent: None, children: BTreeSet::new(), diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index df5f23d..e720538 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -49,10 +49,10 @@ impl CurrentMap { } impl VersionedHashMap { - pub fn new(parent_of_root: Option) -> Self { + pub fn new(parent_of_root: Option, height_of_root: usize) -> Self { VersionedHashMap { parent_of_root, - tree: Tree::new(), + tree: Tree::new(height_of_root), current: None, } } @@ -242,8 +242,8 @@ mod tests { num_nodes: usize, rng: &mut StdRng, ) -> (Tree, VersionedHashMap) { - let mut forward_only_tree = Tree::new(); - let mut versioned_hash_map = VersionedHashMap::new(None); + let mut forward_only_tree = Tree::new(0); + let mut versioned_hash_map = VersionedHashMap::new(None, 0); for i in 1..=num_nodes as CommitId { let parent_commit_id = if i == 1 { @@ -310,8 +310,8 @@ mod tests { #[test] fn test_multiple_roots_err() { - let mut forward_only_tree = Tree::::new(); - let mut versioned_hash_map = VersionedHashMap::::new(None); + let mut forward_only_tree = Tree::::new(0); + let mut versioned_hash_map = VersionedHashMap::::new(None, 0); forward_only_tree.add_root(0, BTreeMap::new()).unwrap(); versioned_hash_map @@ -330,8 +330,8 @@ mod tests { #[test] fn test_commit_id_not_found_err() { - let mut forward_only_tree = Tree::::new(); - let mut versioned_hash_map = VersionedHashMap::::new(None); + let mut forward_only_tree = Tree::::new(0); + let mut versioned_hash_map = VersionedHashMap::::new(None, 0); assert_eq!( forward_only_tree.add_non_root_node(1, 0, BTreeMap::new()), @@ -345,8 +345,8 @@ mod tests { #[test] fn test_commit_id_already_exists_err() { - let mut forward_only_tree = Tree::::new(); - let mut versioned_hash_map = VersionedHashMap::::new(None); + let mut forward_only_tree = Tree::::new(0); + let mut versioned_hash_map = VersionedHashMap::::new(None, 0); forward_only_tree.add_root(0, BTreeMap::new()).unwrap(); versioned_hash_map From 9dc43167061065a24d09a1b9e7e6e4b3597c02c3 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Wed, 25 Sep 2024 12:19:50 +0800 Subject: [PATCH 58/79] tablereader as Arc & pending_part as &'db RwLock --- src/backends/table.rs | 3 ++- src/middlewares/versioned_flat_key_value/mod.rs | 16 ++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/backends/table.rs b/src/backends/table.rs index 4730f04..011d146 100644 --- a/src/backends/table.rs +++ b/src/backends/table.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; use std::fmt::Debug; +use std::sync::Arc; use super::serde::{Decode, Encode, EncodeSubKey}; use super::table_name::TableName; @@ -13,7 +14,7 @@ pub type TableItem<'a, T> = ( Cow<'a, ::Value>, ); pub type TableIter<'a, 'b, T> = Box>>>; -pub type TableReader<'a, T> = Box>; +pub type TableReader<'a, T> = Arc>; #[auto_impl(&, Arc)] pub trait TableRead { diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 578a266..2763397 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -5,6 +5,7 @@ mod table_schema; use std::borrow::Cow; use std::collections::BTreeMap; +use parking_lot::RwLock; pub use pending_part::PendingError; use self::pending_part::pending_schema::PendingKeyValueConfig; @@ -35,21 +36,21 @@ impl HistoryIndices { // struct PendingPart; pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { - pending_part: VersionedHashMap>, + pending_part: &'db RwLock>>, history_index_table: TableReader<'db, HistoryIndicesTable>, commit_id_table: TableReader<'db, CommitIDSchema>, change_history_table: KeyValueStoreBulks<'db, HistoryChangeTable>, } impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { - pub fn get_pending_part(&mut self, commit: CommitID, key: &T::Key) -> Result> { - let res_value = self.pending_part.query(commit, key); + pub fn get_pending_part(&self, commit: CommitID, key: &T::Key) -> Result> { + let res_value = self.pending_part.write().query(commit, key); let history_commit = match res_value { Ok(Some(value)) => { return Ok(value); } Ok(None) => { - if let Some(commit) = self.pending_part.get_parent_of_root() { + if let Some(commit) = self.pending_part.write().get_parent_of_root() { commit } else { return Ok(None); @@ -78,7 +79,10 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { commit: CommitID, updates: BTreeMap>, ) -> Result<()> { - Ok(self.pending_part.add_node(updates, commit, parent_commit)?) + Ok(self + .pending_part + .write() + .add_node(updates, commit, parent_commit)?) } pub fn get_historical_part( @@ -119,7 +123,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { ) -> Result<()> { // old root..=new root's parent let (start_height, confirmed_ids_maps) = - self.pending_part.change_root(new_root_commit_id)?; + self.pending_part.write().change_root(new_root_commit_id)?; for (delta_height, (confirmed_commit_id, updates)) in confirmed_ids_maps.into_iter().enumerate() From 43691164b1641e7fac95d1af680affb3e03e6cf2 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:17:05 +0800 Subject: [PATCH 59/79] move RwLock into pending_part's current --- .../versioned_flat_key_value/mod.rs | 67 ++++++++++++++++--- .../pending_part/commit_tree.rs | 19 ++++++ .../pending_part/versioned_hash_map.rs | 47 +++++++++---- src/traits.rs | 17 +++-- 4 files changed, 122 insertions(+), 28 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 2763397..c2d4501 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -5,7 +5,6 @@ mod table_schema; use std::borrow::Cow; use std::collections::BTreeMap; -use parking_lot::RwLock; pub use pending_part::PendingError; use self::pending_part::pending_schema::PendingKeyValueConfig; @@ -17,6 +16,7 @@ use super::CommitIDSchema; use crate::backends::{TableReader, WriteSchemaTrait}; use crate::errors::Result; use crate::middlewares::{CommitID, HistoryNumber, KeyValueStoreBulks}; +// use crate::traits::{KeyValueStoreBulksTrait, KeyValueStoreManager}; use crate::traits::KeyValueStoreBulksTrait; use crate::StorageError; @@ -36,7 +36,7 @@ impl HistoryIndices { // struct PendingPart; pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { - pending_part: &'db RwLock>>, + pending_part: &'db mut VersionedHashMap>, history_index_table: TableReader<'db, HistoryIndicesTable>, commit_id_table: TableReader<'db, CommitIDSchema>, change_history_table: KeyValueStoreBulks<'db, HistoryChangeTable>, @@ -44,13 +44,13 @@ pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { pub fn get_pending_part(&self, commit: CommitID, key: &T::Key) -> Result> { - let res_value = self.pending_part.write().query(commit, key); + let res_value = self.pending_part.query(commit, key); let history_commit = match res_value { Ok(Some(value)) => { return Ok(value); } Ok(None) => { - if let Some(commit) = self.pending_part.write().get_parent_of_root() { + if let Some(commit) = self.pending_part.get_parent_of_root() { commit } else { return Ok(None); @@ -79,10 +79,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { commit: CommitID, updates: BTreeMap>, ) -> Result<()> { - Ok(self - .pending_part - .write() - .add_node(updates, commit, parent_commit)?) + Ok(self.pending_part.add_node(updates, commit, parent_commit)?) } pub fn get_historical_part( @@ -123,7 +120,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { ) -> Result<()> { // old root..=new root's parent let (start_height, confirmed_ids_maps) = - self.pending_part.write().change_root(new_root_commit_id)?; + self.pending_part.change_root(new_root_commit_id)?; for (delta_height, (confirmed_commit_id, updates)) in confirmed_ids_maps.into_iter().enumerate() @@ -154,3 +151,55 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { Ok(()) } } + +// impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager +// for VersionedStore<'db, T> +// { +// fn iter_historical_changes<'a>( +// &'a self, +// commit_id: &CommitID, +// key: &T::Key, +// ) -> Result)>> { +// let pending_res = self.pending_part.iter_historical_changes(commit_id, key); +// let (history_commit, pending_vec) = match pending_res { +// Ok(pending_vec) => { +// if let Some(commit) = self.pending_part.get_parent_of_root() { +// (commit, pending_vec) +// } else { +// return Ok(pending_vec.into_iter()) +// } +// } +// Err(PendingError::CommitIDNotFound(target_commit)) => { +// assert_eq!(target_commit, commit); +// commit +// } +// Err(other_err) => { +// return Err(StorageError::PendingError(other_err)); +// } +// }; + +// let history_number = if let Some(value) = self.commit_id_table.get(&commit)? { +// value.into_owned() +// } else { +// return Err(StorageError::CommitIDNotFound); +// }; +// self.get_historical_part(history_number, key) + +// } + +// fn discard(self, commit: CommitID) -> Result<()> { +// todo!() +// } + +// fn get_versioned_key(&self, commit: &CommitID, key: &T::Key) -> Result> { +// todo!() +// } + +// fn versioned_iter<'a>( +// &'a self, +// commit: &CommitID, +// key: &T::Key, +// ) -> Result> { +// todo!() +// } +// } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index be2249f..57494d1 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -202,6 +202,25 @@ impl Tree { } } +impl Tree { + #[allow(clippy::type_complexity)] + pub fn iter_historical_changes( + &self, + commit_id: &S::CommitId, + key: &S::Key, + ) -> PendResult)>, S> { + let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); + let mut path = Vec::new(); + while let Some(node) = node_option { + if let Some(RecoverRecord { value, .. }) = node.modifications.get(key) { + path.push((node.commit_id, value.clone())); + } + node_option = self.get_parent_node(node); + } + Ok(path) + } +} + impl Tree { pub fn get_apply_map_from_root_included( &self, diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index e720538..96310e4 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -9,10 +9,12 @@ use super::{ PendingError, }; +use parking_lot::RwLock; + pub struct VersionedHashMap { parent_of_root: Option, tree: Tree, - current: Option>, + current: RwLock>>, } struct CurrentMap { @@ -53,7 +55,7 @@ impl VersionedHashMap { VersionedHashMap { parent_of_root, tree: Tree::new(height_of_root), - current: None, + current: RwLock::new(None), } } } @@ -70,7 +72,7 @@ impl VersionedHashMap { if let Some(parent_of_new_root) = to_commit.last() { self.parent_of_root = Some(parent_of_new_root.0); - self.current = None; + *self.current.write() = None; } Ok((start_height_to_commit, to_commit)) @@ -98,7 +100,7 @@ impl VersionedHashMap { } fn add_root(&mut self, updates: KeyValueMap, commit_id: S::CommitId) -> PendResult<(), S> { - if self.current.is_some() { + if self.current.read().is_some() { return Err(PendingError::MultipleRootsNotAllowed); } @@ -131,7 +133,8 @@ impl VersionedHashMap { self.checkout_current(parent_commit_id)?; // add node to tree - let current = self.current.as_ref().unwrap(); + let current_read = self.current.read(); + let current = current_read.as_ref().unwrap(); let mut modifications = BTreeMap::new(); for (key, value) in updates.into_iter() { let last_commit_id = current.map.get(&key).map(|s| s.commit_id); @@ -155,7 +158,7 @@ impl VersionedHashMap { // Some(None): pending_part know that this key has been deleted // Some(Some(value)): pending_part know this key's value pub fn query( - &mut self, + &self, commit_id: S::CommitId, key: &S::Key, ) -> PendResult>, S> { @@ -165,6 +168,7 @@ impl VersionedHashMap { // query Ok(self .current + .read() .as_ref() .unwrap() .map @@ -174,11 +178,25 @@ impl VersionedHashMap { } impl VersionedHashMap { - fn checkout_current(&mut self, target_commit_id: S::CommitId) -> PendResult<(), S> { - if let Some(ref mut current) = self.current { + #[allow(clippy::type_complexity)] + pub fn iter_historical_changes( + &self, + commit_id: &S::CommitId, + key: &S::Key, + ) -> PendResult)>, S> { + self.tree.iter_historical_changes(commit_id, key) + } +} + +impl VersionedHashMap { + fn checkout_current(&self, target_commit_id: S::CommitId) -> PendResult<(), S> { + let current_commit_id = self.current.read().as_ref().map(|c| c.commit_id); + if let Some(current_commit_id) = current_commit_id { let (rollbacks, applys) = self .tree - .collect_rollback_and_apply_ops(current.commit_id, target_commit_id)?; + .collect_rollback_and_apply_ops(current_commit_id, target_commit_id)?; + let mut current_option = self.current.write(); + let current = current_option.as_mut().unwrap(); current.rollback(rollbacks); current.apply(applys); current.commit_id = target_commit_id; @@ -188,10 +206,13 @@ impl VersionedHashMap { .tree .get_apply_map_from_root_included(target_commit_id)?; current.apply(applys); - self.current = Some(current) - }; + *self.current.write() = Some(current); + } - assert_eq!(self.current.as_ref().unwrap().commit_id, target_commit_id); + assert_eq!( + self.current.read().as_ref().unwrap().commit_id, + target_commit_id + ); Ok(()) } @@ -293,7 +314,7 @@ mod tests { ]; let mut rng = StdRng::from_seed(seed); - let (forward_only_tree, mut versioned_hash_map) = generate_random_tree(num_nodes, &mut rng); + let (forward_only_tree, versioned_hash_map) = generate_random_tree(num_nodes, &mut rng); for _ in 0..100 { let commit_id = rng.gen_range(1..=num_nodes) as CommitId; for ikey in 0..10 { diff --git a/src/traits.rs b/src/traits.rs index 0919113..798c58a 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -19,23 +19,28 @@ where V: 'static, C: 'static, { - type Store: KeyValueStore; + //type Store: KeyValueStore; /// Get the key value store after the commit of given id - fn get_versioned_store(&self, commit: &C) -> Result; + //fn get_versioned_store(&self, commit: &C) -> Result; + + /// iter key at the key value store after the commit of given id, only FlatKV requires + fn versioned_iter<'a>( + &'a self, + commit: &C, + key: &K, + ) -> Result>; /// Start from the given commit, and iter changes backforward fn iter_historical_changes<'a>( &'a self, commit_id: &C, key: &K, - ) -> Result>; + ) -> Result)>>; fn discard(self, commit: C) -> Result<()>; - fn get_versioned_key(&self, commit: &C, key: &K) -> Result> { - self.get_versioned_store(commit)?.get(key) - } + fn get_versioned_key(&self, commit: &C, key: &K) -> Result>; } pub trait KeyValueStoreBulksTrait { From fa9f914609cdfa82148e50f39610aeeb42489231 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:56:23 +0800 Subject: [PATCH 60/79] pending part iter_historical_changes iterator& --- .../pending_part/commit_tree.rs | 11 +++++------ .../pending_part/versioned_hash_map.rs | 7 +++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 57494d1..02b079b 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -203,21 +203,20 @@ impl Tree { } impl Tree { - #[allow(clippy::type_complexity)] - pub fn iter_historical_changes( - &self, + pub fn iter_historical_changes<'a>( + &'a self, commit_id: &S::CommitId, key: &S::Key, - ) -> PendResult)>, S> { + ) -> PendResult)>, S> { let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); let mut path = Vec::new(); while let Some(node) = node_option { if let Some(RecoverRecord { value, .. }) = node.modifications.get(key) { - path.push((node.commit_id, value.clone())); + path.push((&node.commit_id, value)); } node_option = self.get_parent_node(node); } - Ok(path) + Ok(path.into_iter()) } } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 96310e4..a5a05a1 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -178,12 +178,11 @@ impl VersionedHashMap { } impl VersionedHashMap { - #[allow(clippy::type_complexity)] - pub fn iter_historical_changes( - &self, + pub fn iter_historical_changes<'a>( + &'a self, commit_id: &S::CommitId, key: &S::Key, - ) -> PendResult)>, S> { + ) -> PendResult)>, S> { self.tree.iter_historical_changes(commit_id, key) } } From f8a5c78a2768668e206da999d0dff5ade7c406e4 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:31:13 +0800 Subject: [PATCH 61/79] impl iter_historical_changes_history_part --- src/backends/impls/in_memory_db.rs | 3 +- src/backends/table_name.rs | 3 + src/errors.rs | 2 +- src/middlewares/commit_id_schema.rs | 9 + .../versioned_flat_key_value/mod.rs | 159 ++++++++++++------ 5 files changed, 123 insertions(+), 53 deletions(-) diff --git a/src/backends/impls/in_memory_db.rs b/src/backends/impls/in_memory_db.rs index 0db5f65..34584c4 100644 --- a/src/backends/impls/in_memory_db.rs +++ b/src/backends/impls/in_memory_db.rs @@ -33,7 +33,8 @@ impl<'b, T: TableSchema> TableRead for InMemoryTable<'b> { fn iter(&self, key: &T::Key) -> Result> { let range = self.inner.0.range((self.col, key.encode().into_owned())..); let iter = range - .filter(|((col, _), _)| *col == self.col) + //.filter(|((col, _), _)| *col == self.col) + .take_while(move |((col, _), _)| *col == self.col) .map(|((_, k), v)| Ok((::decode(k)?, ::decode(v)?))); Ok(Box::new(iter)) } diff --git a/src/backends/table_name.rs b/src/backends/table_name.rs index 734999f..62e29d6 100644 --- a/src/backends/table_name.rs +++ b/src/backends/table_name.rs @@ -1,6 +1,7 @@ #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum TableName { CommitID, + HistoryNumber, HistoryChange(VersionedKVName), HistoryIndex(VersionedKVName), #[cfg(test)] @@ -33,6 +34,7 @@ impl From for u32 { }; match t { CommitID => make_id(), + HistoryNumber => make_id(), HistoryChange(FlatKV) => make_id(), HistoryIndex(FlatKV) => make_id(), #[cfg(test)] @@ -45,6 +47,7 @@ impl From for &'static str { fn from(t: TableName) -> Self { match t { CommitID => "commit_id", + HistoryNumber => "history_number", HistoryChange(FlatKV) => "flat_kv_change_history", HistoryIndex(FlatKV) => "flat_kv_history_index", #[cfg(test)] diff --git a/src/errors.rs b/src/errors.rs index 8be05b7..3644abf 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -22,7 +22,7 @@ pub enum StorageError { pub type Result = ::std::result::Result; -#[derive(Error, Debug)] +#[derive(Error, Debug, Clone, Copy)] pub enum DecodeError { #[error("incorrect input length")] IncorrectLength, diff --git a/src/middlewares/commit_id_schema.rs b/src/middlewares/commit_id_schema.rs index 36f626b..cf6efbe 100644 --- a/src/middlewares/commit_id_schema.rs +++ b/src/middlewares/commit_id_schema.rs @@ -20,3 +20,12 @@ impl TableSchema for CommitIDSchema { type Key = CommitID; type Value = HistoryNumber; } + +#[derive(Clone, Copy)] +pub struct HistoryNumberSchema; + +impl TableSchema for HistoryNumberSchema { + const NAME: TableName = TableName::HistoryNumber; + type Key = HistoryNumber; + type Value = CommitID; +} diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index c2d4501..5ed93ed 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -11,6 +11,7 @@ use self::pending_part::pending_schema::PendingKeyValueConfig; use self::table_schema::{HistoryChangeTable, HistoryIndicesTable, VersionedKeyValueSchema}; use pending_part::VersionedHashMap; +use super::commit_id_schema::HistoryNumberSchema; use super::ChangeKey; use super::CommitIDSchema; use crate::backends::{TableReader, WriteSchemaTrait}; @@ -39,6 +40,7 @@ pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { pending_part: &'db mut VersionedHashMap>, history_index_table: TableReader<'db, HistoryIndicesTable>, commit_id_table: TableReader<'db, CommitIDSchema>, + history_number_table: TableReader<'db, HistoryNumberSchema>, change_history_table: KeyValueStoreBulks<'db, HistoryChangeTable>, } @@ -136,6 +138,12 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { ); write_schema.write::(commit_id_table_op); + let history_number_table_op = ( + Cow::Owned(history_number), + Some(Cow::Owned(confirmed_commit_id)), + ); + write_schema.write::(history_number_table_op); + let history_indices_table_op = updates.keys().map(|key| { ( Cow::Owned(HistoryIndexKey(key.clone(), history_number)), @@ -152,54 +160,103 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { } } -// impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager -// for VersionedStore<'db, T> -// { -// fn iter_historical_changes<'a>( -// &'a self, -// commit_id: &CommitID, -// key: &T::Key, -// ) -> Result)>> { -// let pending_res = self.pending_part.iter_historical_changes(commit_id, key); -// let (history_commit, pending_vec) = match pending_res { -// Ok(pending_vec) => { -// if let Some(commit) = self.pending_part.get_parent_of_root() { -// (commit, pending_vec) -// } else { -// return Ok(pending_vec.into_iter()) -// } -// } -// Err(PendingError::CommitIDNotFound(target_commit)) => { -// assert_eq!(target_commit, commit); -// commit -// } -// Err(other_err) => { -// return Err(StorageError::PendingError(other_err)); -// } -// }; - -// let history_number = if let Some(value) = self.commit_id_table.get(&commit)? { -// value.into_owned() -// } else { -// return Err(StorageError::CommitIDNotFound); -// }; -// self.get_historical_part(history_number, key) - -// } - -// fn discard(self, commit: CommitID) -> Result<()> { -// todo!() -// } - -// fn get_versioned_key(&self, commit: &CommitID, key: &T::Key) -> Result> { -// todo!() -// } - -// fn versioned_iter<'a>( -// &'a self, -// commit: &CommitID, -// key: &T::Key, -// ) -> Result> { -// todo!() -// } -// } +impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { + // impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager + // for VersionedStore<'db, T> + // { + fn iter_historical_changes_history_part<'a>( + &'a self, + commit_id: &CommitID, + key: &'a T::Key, + ) -> Result)>> { + let query_number = if let Some(value) = self.commit_id_table.get(commit_id)? { + value.into_owned() + } else { + return Err(StorageError::CommitIDNotFound); + }; + + let mut num_items = 0; + let range_query_key = HistoryIndexKey(key.clone(), query_number); + for item in self.history_index_table.iter(&range_query_key)? { + let (k_with_history_number, indices) = item?; + let HistoryIndexKey(k, history_number) = k_with_history_number.as_ref(); + if k != key { + break; + } else { + let found_version_number = indices.as_ref().last(*history_number); + let found_value = self + .change_history_table + .get_versioned_key(&found_version_number, key)?; + let found_commit_id = self.history_number_table.get(&found_version_number)?; + num_items += 1; + } + } + + let history_iter = self + .history_index_table + .iter(&range_query_key)? + .take(num_items) + .map(move |item| { + let (k_with_history_number, indices) = + item.expect("previous for + take() should truncate before err"); + let HistoryIndexKey(k, history_number) = k_with_history_number.as_ref(); + let found_version_number = indices.as_ref().last(*history_number); + let found_value = self + .change_history_table + .get_versioned_key(&found_version_number, key) + .expect("previous for + take() should truncate before err"); + let found_commit_id = self + .history_number_table + .get(&found_version_number) + .expect("previous for + take() should truncate before err"); + (found_commit_id.unwrap().into_owned(), key, found_value) + }); + Ok(history_iter) + } + + // fn iter_historical_changes<'a>( + // &'a self, + // commit_id: &CommitID, + // key: &T::Key, + // ) -> Result)>> { + // let pending_res = self.pending_part.iter_historical_changes(commit_id, key); + // let (history_commit, pending_iter) = match pending_res { + // Ok(pending_iter) => { + // if let Some(history_commit) = self.pending_part.get_parent_of_root() { + // (history_commit, pending_iter) + // } else { + // return Ok(pending_iter) + // } + // } + // Err(PendingError::CommitIDNotFound(target_commit)) => { + // assert_eq!(target_commit, *commit_id); + // (target_commit, Default::default()) + // } + // }; + + // let history_number = if let Some(value) = self.commit_id_table.get(&commit)? { + // value.into_owned() + // } else { + // return Err(StorageError::CommitIDNotFound); + // }; + // self.get_historical_part(history_number, key) + + // } + + // fn discard(self, commit: CommitID) -> Result<()> { + // todo!() + // } + + // fn get_versioned_key(&self, commit: &CommitID, key: &T::Key) -> Result> { + // todo!() + // } + + // fn versioned_iter<'a>( + // &'a self, + // commit: &CommitID, + // key: &T::Key, + // ) -> Result> { + // todo!() + // } + // } +} From b4e45e7bddba644fcefe57e8f40a0862959678ae Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:44:32 +0800 Subject: [PATCH 62/79] unify return type of iter history&pending changes --- .../versioned_flat_key_value/pending_part/commit_tree.rs | 6 +++--- .../pending_part/versioned_hash_map.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 02b079b..420d59e 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -206,13 +206,13 @@ impl Tree { pub fn iter_historical_changes<'a>( &'a self, commit_id: &S::CommitId, - key: &S::Key, - ) -> PendResult)>, S> { + key: &'a S::Key, + ) -> PendResult)>, S> { let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); let mut path = Vec::new(); while let Some(node) = node_option { if let Some(RecoverRecord { value, .. }) = node.modifications.get(key) { - path.push((&node.commit_id, value)); + path.push((node.commit_id, key, value.clone())); } node_option = self.get_parent_node(node); } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index a5a05a1..30c1298 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -181,8 +181,8 @@ impl VersionedHashMap { pub fn iter_historical_changes<'a>( &'a self, commit_id: &S::CommitId, - key: &S::Key, - ) -> PendResult)>, S> { + key: &'a S::Key, + ) -> PendResult)>, S> { self.tree.iter_historical_changes(commit_id, key) } } From d17811afcf1447295092613eb3e381115c8e32a2 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:16:03 +0800 Subject: [PATCH 63/79] impl iter_historical_changes --- .../versioned_flat_key_value/mod.rs | 59 +++++++++---------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 5ed93ed..bc81de0 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -161,9 +161,6 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { } impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { - // impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager - // for VersionedStore<'db, T> - // { fn iter_historical_changes_history_part<'a>( &'a self, commit_id: &CommitID, @@ -213,35 +210,35 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { }); Ok(history_iter) } +} - // fn iter_historical_changes<'a>( - // &'a self, - // commit_id: &CommitID, - // key: &T::Key, - // ) -> Result)>> { - // let pending_res = self.pending_part.iter_historical_changes(commit_id, key); - // let (history_commit, pending_iter) = match pending_res { - // Ok(pending_iter) => { - // if let Some(history_commit) = self.pending_part.get_parent_of_root() { - // (history_commit, pending_iter) - // } else { - // return Ok(pending_iter) - // } - // } - // Err(PendingError::CommitIDNotFound(target_commit)) => { - // assert_eq!(target_commit, *commit_id); - // (target_commit, Default::default()) - // } - // }; - - // let history_number = if let Some(value) = self.commit_id_table.get(&commit)? { - // value.into_owned() - // } else { - // return Err(StorageError::CommitIDNotFound); - // }; - // self.get_historical_part(history_number, key) - - // } +impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { + // impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager + // for VersionedStore<'db, T> + // { + fn iter_historical_changes<'a>( + &'a self, + commit_id: &CommitID, + key: &'a T::Key, + ) -> Result)>>> { + let pending_res = self.pending_part.iter_historical_changes(commit_id, key); + match pending_res { + Ok(pending_iter) => { + if let Some(history_commit) = self.pending_part.get_parent_of_root() { + let history_iter = self.iter_historical_changes_history_part(&history_commit, key)?; + return Ok(Box::new(pending_iter.chain(history_iter))) + } else { + return Ok(Box::new(pending_iter)) + } + } + Err(PendingError::CommitIDNotFound(target_commit)) => { + assert_eq!(target_commit, *commit_id); + let history_iter = self.iter_historical_changes_history_part(&target_commit, key)?; + return Ok(Box::new(history_iter)) + } + _ => unreachable!("pending_part's iter_historical_changes() does not return other errors") + }; + } // fn discard(self, commit: CommitID) -> Result<()> { // todo!() From 6dbd856eeedd6cdbc6ad3dbeca349efcc170c386 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:46:57 +0800 Subject: [PATCH 64/79] impl get_versioned_key --- .../versioned_flat_key_value/mod.rs | 78 ++++++++++--------- .../pending_part/commit_tree.rs | 15 ++++ .../pending_part/versioned_hash_map.rs | 15 +++- 3 files changed, 69 insertions(+), 39 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index bc81de0..6032b6a 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -45,34 +45,12 @@ pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { } impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { - pub fn get_pending_part(&self, commit: CommitID, key: &T::Key) -> Result> { - let res_value = self.pending_part.query(commit, key); - let history_commit = match res_value { - Ok(Some(value)) => { - return Ok(value); - } - Ok(None) => { - if let Some(commit) = self.pending_part.get_parent_of_root() { - commit - } else { - return Ok(None); - } - } - Err(PendingError::CommitIDNotFound(target_commit)) => { - assert_eq!(target_commit, commit); - commit - } - Err(other_err) => { - return Err(StorageError::PendingError(other_err)); - } - }; - - let history_number = if let Some(value) = self.commit_id_table.get(&commit)? { - value.into_owned() + fn get_history_number_by_commit_id(&self, commit: CommitID) -> Result { + if let Some(value) = self.commit_id_table.get(&commit)? { + Ok(value.into_owned()) } else { - return Err(StorageError::CommitIDNotFound); - }; - self.get_historical_part(history_number, key) + Err(StorageError::CommitIDNotFound) + } } pub fn add_to_pending_part( @@ -216,6 +194,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { // impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager // for VersionedStore<'db, T> // { + #[allow(clippy::type_complexity)] fn iter_historical_changes<'a>( &'a self, commit_id: &CommitID, @@ -225,28 +204,53 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { match pending_res { Ok(pending_iter) => { if let Some(history_commit) = self.pending_part.get_parent_of_root() { - let history_iter = self.iter_historical_changes_history_part(&history_commit, key)?; - return Ok(Box::new(pending_iter.chain(history_iter))) + let history_iter = + self.iter_historical_changes_history_part(&history_commit, key)?; + Ok(Box::new(pending_iter.chain(history_iter))) } else { - return Ok(Box::new(pending_iter)) + Ok(Box::new(pending_iter)) } } Err(PendingError::CommitIDNotFound(target_commit)) => { assert_eq!(target_commit, *commit_id); - let history_iter = self.iter_historical_changes_history_part(&target_commit, key)?; - return Ok(Box::new(history_iter)) + let history_iter = + self.iter_historical_changes_history_part(&target_commit, key)?; + Ok(Box::new(history_iter)) } - _ => unreachable!("pending_part's iter_historical_changes() does not return other errors") - }; + Err(other_err) => Err(StorageError::PendingError(other_err)), + } } // fn discard(self, commit: CommitID) -> Result<()> { // todo!() // } - // fn get_versioned_key(&self, commit: &CommitID, key: &T::Key) -> Result> { - // todo!() - // } + fn get_versioned_key(&self, commit: &CommitID, key: &T::Key) -> Result> { + // let pending_res = self.pending_part.get_versioned_key(commit, key); // this will checkout_current + let pending_res = self.pending_part.get_versioned_key(commit, key); // this does not checkout_current + let history_commit = match pending_res { + Ok(Some(value)) => { + return Ok(value); + } + Ok(None) => { + if let Some(commit) = self.pending_part.get_parent_of_root() { + commit + } else { + return Ok(None); + } + } + Err(PendingError::CommitIDNotFound(target_commit)) => { + assert_eq!(target_commit, *commit); + target_commit + } + Err(other_err) => { + return Err(StorageError::PendingError(other_err)); + } + }; + + let history_number = self.get_history_number_by_commit_id(history_commit)?; + self.get_historical_part(history_number, key) + } // fn versioned_iter<'a>( // &'a self, diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 420d59e..a315787 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -218,6 +218,21 @@ impl Tree { } Ok(path.into_iter()) } + + pub fn get_versioned_key( + &self, + commit_id: &S::CommitId, + key: &S::Key, + ) -> PendResult>, S> { + let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); + while let Some(node) = node_option { + if let Some(RecoverRecord { value, .. }) = node.modifications.get(key) { + return Ok(Some(value.clone())); + } + node_option = self.get_parent_node(node); + } + Ok(None) + } } impl Tree { diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 30c1298..b778b70 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -157,7 +157,7 @@ impl VersionedHashMap { // None: pending_part not know // Some(None): pending_part know that this key has been deleted // Some(Some(value)): pending_part know this key's value - pub fn query( + pub fn query_frequent_commit_id( &self, commit_id: S::CommitId, key: &S::Key, @@ -177,6 +177,7 @@ impl VersionedHashMap { } } +// only one key, no need to invoke checkout_current impl VersionedHashMap { pub fn iter_historical_changes<'a>( &'a self, @@ -185,6 +186,14 @@ impl VersionedHashMap { ) -> PendResult)>, S> { self.tree.iter_historical_changes(commit_id, key) } + + pub fn get_versioned_key( + &self, + commit_id: &S::CommitId, + key: &S::Key, + ) -> PendResult>, S> { + self.tree.get_versioned_key(commit_id, key) + } } impl VersionedHashMap { @@ -318,7 +327,9 @@ mod tests { let commit_id = rng.gen_range(1..=num_nodes) as CommitId; for ikey in 0..10 { let key: u64 = ikey; - let result = versioned_hash_map.query(commit_id, &key).unwrap(); + let result = versioned_hash_map + .query_frequent_commit_id(commit_id, &key) + .unwrap(); let apply_map = forward_only_tree .get_apply_map_from_root_included(commit_id) .unwrap(); From 2f5392e150ab3157023a5a432d2adc268c098c87 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:46:32 +0800 Subject: [PATCH 65/79] impl pending_part's discard --- .../pending_part/commit_tree.rs | 25 ++++++++++++++++++- .../pending_part/versioned_hash_map.rs | 11 ++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index a315787..4f1255f 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -41,7 +41,7 @@ impl Tree { } } - fn contains_commit_id(&self, commit_id: &S::CommitId) -> bool { + pub(super) fn contains_commit_id(&self, commit_id: &S::CommitId) -> bool { self.index_map.contains_key(commit_id) } @@ -57,6 +57,10 @@ impl Tree { &self.nodes[slab_index] } + fn get_mut_node_by_slab_index(&mut self, slab_index: SlabIndex) -> &mut TreeNode { + &mut self.nodes[slab_index] + } + fn get_node_by_commit_id(&self, commit_id: S::CommitId) -> PendResult<&TreeNode, S> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; Ok(self.get_node_by_slab_index(slab_index)) @@ -233,6 +237,25 @@ impl Tree { } Ok(None) } + + // true: is_root, false: is_not_root + pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult { + let slab_index = self.get_slab_index_by_commit_id(commit_id)?; + let node = self.get_node_by_slab_index(slab_index); + if let Some(parent_of_discard) = node.parent { + let to_remove = self.bfs_subtree(slab_index); + for idx in to_remove { + self.index_map.remove(&self.nodes.remove(idx).commit_id); + } + let parent_node = self.get_mut_node_by_slab_index(parent_of_discard); + parent_node.children.remove(&slab_index); + Ok(false) + } else { + self.nodes.clear(); + self.index_map.clear(); + Ok(true) + } + } } impl Tree { diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index b778b70..95c4301 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -194,6 +194,17 @@ impl VersionedHashMap { ) -> PendResult>, S> { self.tree.get_versioned_key(commit_id, key) } + + pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult { + let is_root = self.tree.discard(commit_id)?; + let current_commit_id = self.current.read().as_ref().map(|c| c.commit_id); + if let Some(current_commit_id) = current_commit_id { + if !self.tree.contains_commit_id(¤t_commit_id) { + *self.current.write() = None; + } + } + Ok(is_root) + } } impl VersionedHashMap { From a27a8da64ba26ba32190645636de0d2b7cc9e17e Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Sun, 29 Sep 2024 10:37:13 +0800 Subject: [PATCH 66/79] impl discard for VersionedStore --- src/middlewares/versioned_flat_key_value/mod.rs | 6 +++--- .../pending_part/commit_tree.rs | 12 +++++++----- .../versioned_flat_key_value/pending_part/error.rs | 2 ++ .../pending_part/versioned_hash_map.rs | 10 +++++++--- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 6032b6a..50ad639 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -221,9 +221,9 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { } } - // fn discard(self, commit: CommitID) -> Result<()> { - // todo!() - // } + fn discard(&mut self, commit: CommitID) -> Result<()> { + Ok(self.pending_part.discard(commit)?) + } fn get_versioned_key(&self, commit: &CommitID, key: &T::Key) -> Result> { // let pending_res = self.pending_part.get_versioned_key(commit, key); // this will checkout_current diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index 4f1255f..df21358 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -41,6 +41,10 @@ impl Tree { } } + pub(super) fn get_height_of_root(&self) -> usize { + self.height_of_root + } + pub(super) fn contains_commit_id(&self, commit_id: &S::CommitId) -> bool { self.index_map.contains_key(commit_id) } @@ -239,7 +243,7 @@ impl Tree { } // true: is_root, false: is_not_root - pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult { + pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult<(), S> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; let node = self.get_node_by_slab_index(slab_index); if let Some(parent_of_discard) = node.parent { @@ -249,11 +253,9 @@ impl Tree { } let parent_node = self.get_mut_node_by_slab_index(parent_of_discard); parent_node.children.remove(&slab_index); - Ok(false) + Ok(()) } else { - self.nodes.clear(); - self.index_map.clear(); - Ok(true) + Err(PendingError::RootShouldNotBeDiscarded) } } } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/error.rs b/src/middlewares/versioned_flat_key_value/pending_part/error.rs index e7f6bca..55d5efd 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/error.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/error.rs @@ -12,4 +12,6 @@ pub enum PendingError { CommitIdAlreadyExists(CommitId), #[error("non_root node should have parent")] NonRootNodeHasNoParentError, + #[error("pending root should not be discarded")] + RootShouldNotBeDiscarded, } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 95c4301..e32dc96 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -84,6 +84,10 @@ impl VersionedHashMap { self.parent_of_root } + pub fn get_height_of_root(&self) -> usize { + self.tree.get_height_of_root() + } + pub fn add_node( &mut self, updates: KeyValueMap, @@ -195,15 +199,15 @@ impl VersionedHashMap { self.tree.get_versioned_key(commit_id, key) } - pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult { - let is_root = self.tree.discard(commit_id)?; + pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult<(), S> { + self.tree.discard(commit_id)?; let current_commit_id = self.current.read().as_ref().map(|c| c.commit_id); if let Some(current_commit_id) = current_commit_id { if !self.tree.contains_commit_id(¤t_commit_id) { *self.current.write() = None; } } - Ok(is_root) + Ok(()) } } From 0fa7168ccb709c763a1d964e74a8e713f03bf65f Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Sun, 29 Sep 2024 15:36:44 +0800 Subject: [PATCH 67/79] OneStore for VersionedStore: KeyValueStoreManager --- .../versioned_flat_key_value/mod.rs | 45 ++++++++++++------- src/traits.rs | 17 +++---- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 50ad639..b17985c 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -4,6 +4,7 @@ mod table_schema; use std::borrow::Cow; use std::collections::BTreeMap; +use std::marker::PhantomData; pub use pending_part::PendingError; @@ -17,8 +18,7 @@ use super::CommitIDSchema; use crate::backends::{TableReader, WriteSchemaTrait}; use crate::errors::Result; use crate::middlewares::{CommitID, HistoryNumber, KeyValueStoreBulks}; -// use crate::traits::{KeyValueStoreBulksTrait, KeyValueStoreManager}; -use crate::traits::KeyValueStoreBulksTrait; +use crate::traits::{KeyValueStoreBulksTrait, KeyValueStoreManager, KeyValueStore}; use crate::StorageError; #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] @@ -190,10 +190,34 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { } } -impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { - // impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager - // for VersionedStore<'db, T> - // { +pub struct OneStore { + map: BTreeMap::, + _marker: PhantomData, +} + +impl KeyValueStore for OneStore { + fn get(&self, key: &K) -> Result> { + Ok(self.map.get(key).map(|v| v.clone())) + } + + fn iter<'a>(&'a self, key: &K) -> Result> { + Ok(self.map.range(key..)) + } + + fn commit(self, commit: C, changes: impl Iterator) { + todo!() + } +} + +// impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { +impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager + for VersionedStore<'db, T> +{ + type Store = OneStore; + fn get_versioned_store(&self, commit: &CommitID) -> Result { + todo!() + } + #[allow(clippy::type_complexity)] fn iter_historical_changes<'a>( &'a self, @@ -251,13 +275,4 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { let history_number = self.get_history_number_by_commit_id(history_commit)?; self.get_historical_part(history_number, key) } - - // fn versioned_iter<'a>( - // &'a self, - // commit: &CommitID, - // key: &T::Key, - // ) -> Result> { - // todo!() - // } - // } } diff --git a/src/traits.rs b/src/traits.rs index 798c58a..af236d5 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -19,26 +19,19 @@ where V: 'static, C: 'static, { - //type Store: KeyValueStore; + type Store: KeyValueStore; /// Get the key value store after the commit of given id - //fn get_versioned_store(&self, commit: &C) -> Result; - - /// iter key at the key value store after the commit of given id, only FlatKV requires - fn versioned_iter<'a>( - &'a self, - commit: &C, - key: &K, - ) -> Result>; + fn get_versioned_store(&self, commit: &C) -> Result; /// Start from the given commit, and iter changes backforward fn iter_historical_changes<'a>( &'a self, commit_id: &C, - key: &K, - ) -> Result)>>; + key: &'a K, + ) -> Result)>>>; - fn discard(self, commit: C) -> Result<()>; + fn discard(&mut self, commit: C) -> Result<()>; fn get_versioned_key(&self, commit: &C, key: &K) -> Result>; } From bc3e7ea6191eaf0ba5d87bd9fb0619d642b9c8e2 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Sun, 29 Sep 2024 16:35:19 +0800 Subject: [PATCH 68/79] impl pending part's get_versioned_store --- .../versioned_flat_key_value/mod.rs | 31 +++++++++++++++---- .../pending_part/versioned_hash_map.rs | 18 +++++++++++ src/traits.rs | 1 + 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index b17985c..2b0951f 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -18,7 +18,7 @@ use super::CommitIDSchema; use crate::backends::{TableReader, WriteSchemaTrait}; use crate::errors::Result; use crate::middlewares::{CommitID, HistoryNumber, KeyValueStoreBulks}; -use crate::traits::{KeyValueStoreBulksTrait, KeyValueStoreManager, KeyValueStore}; +use crate::traits::{KeyValueStore, KeyValueStoreBulksTrait, KeyValueStoreManager}; use crate::StorageError; #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] @@ -191,13 +191,24 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { } pub struct OneStore { - map: BTreeMap::, + map: BTreeMap, _marker: PhantomData, } -impl KeyValueStore for OneStore { +impl OneStore { + pub fn from_map(map: BTreeMap) -> Self { + OneStore { + map, + _marker: PhantomData, + } + } +} + +impl KeyValueStore + for OneStore +{ fn get(&self, key: &K) -> Result> { - Ok(self.map.get(key).map(|v| v.clone())) + Ok(self.map.get(key).cloned()) } fn iter<'a>(&'a self, key: &K) -> Result> { @@ -209,13 +220,21 @@ impl KeyValueStore fo } } -// impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager for VersionedStore<'db, T> { type Store = OneStore; fn get_versioned_store(&self, commit: &CommitID) -> Result { - todo!() + let pending_res = self.pending_part.get_versioned_store(*commit); + match pending_res { + Ok(pending_map) => Ok(OneStore::from_map(pending_map)), + Err(PendingError::CommitIDNotFound(target_commit_id)) => { + assert_eq!(target_commit_id, *commit); + let history_map: BTreeMap = BTreeMap::new(); //todo + Ok(OneStore::from_map(history_map)) + } + Err(other_err) => Err(StorageError::PendingError(other_err)), + } } #[allow(clippy::type_complexity)] diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index e32dc96..ed908e2 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -209,6 +209,24 @@ impl VersionedHashMap { } Ok(()) } + + pub fn get_versioned_store( + &self, + commit_id: S::CommitId, + ) -> PendResult, S> { + self.checkout_current(commit_id)?; + let current_read = self.current.read(); + let map: BTreeMap<_, _> = current_read + .as_ref() + .unwrap() + .map + .iter() + .filter_map(|(k, apply_record)| { + apply_record.value.as_ref().map(|v| (k.clone(), v.clone())) + }) + .collect(); + Ok(map) + } } impl VersionedHashMap { diff --git a/src/traits.rs b/src/traits.rs index af236d5..5856e81 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -25,6 +25,7 @@ where fn get_versioned_store(&self, commit: &C) -> Result; /// Start from the given commit, and iter changes backforward + #[allow(clippy::type_complexity)] fn iter_historical_changes<'a>( &'a self, commit_id: &C, From c402d992219ec797b091d6cdaa674374f8d36a57 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Sun, 29 Sep 2024 18:51:02 +0800 Subject: [PATCH 69/79] impl get_versioned_store for VersionedStore --- .../versioned_flat_key_value/mod.rs | 74 ++++++++++++++++--- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 2b0951f..f3f91a0 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -6,6 +6,7 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::marker::PhantomData; +use parking_lot::RwLock; pub use pending_part::PendingError; use self::pending_part::pending_schema::PendingKeyValueConfig; @@ -34,7 +35,7 @@ impl HistoryIndices { } } -// struct PendingPart; +const MIN_HISTORY_NUMBER_MINUS_ONE: HistoryNumber = 0; pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { pending_part: &'db mut VersionedHashMap>, @@ -42,6 +43,7 @@ pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { commit_id_table: TableReader<'db, CommitIDSchema>, history_number_table: TableReader<'db, HistoryNumberSchema>, change_history_table: KeyValueStoreBulks<'db, HistoryChangeTable>, + history_min_key: RwLock>, } impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { @@ -69,7 +71,6 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { ) -> Result> { let range_query_key = HistoryIndexKey(key.clone(), query_version_number); - // history_number should be decreasing let found_version_number = match self.history_index_table.iter(&range_query_key)?.next() { None => { return Ok(None); @@ -90,6 +91,28 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { self.change_history_table .get_versioned_key(&found_version_number, key) } + + pub fn find_larger_key(&self, key: &T::Key) -> Result> { + // todo: here correct? + let range_query_key = HistoryIndexKey(key.clone(), MIN_HISTORY_NUMBER_MINUS_ONE); + + match self.history_index_table.iter(&range_query_key)?.next() { + None => Ok(None), + Some(Err(e)) => Err(e.into()), + Some(Ok((k, _))) if &k.as_ref().0 > key => Ok(Some(k.as_ref().0.to_owned())), + _ => unreachable!( + "iter((key, MIN_HISTORY_NUMBER_MINUS_ONE)) should not result in k <= key" + ), + } + } +} + +fn need_update_min_key(original_min: Option<&K>, challenge_min: &K) -> bool { + if let Some(original_min) = original_min { + original_min > challenge_min + } else { + true + } } impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { @@ -102,11 +125,13 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { let (start_height, confirmed_ids_maps) = self.pending_part.change_root(new_root_commit_id)?; + let mut confirmed_min_key = None; + for (delta_height, (confirmed_commit_id, updates)) in confirmed_ids_maps.into_iter().enumerate() { let height = (start_height + delta_height) as u64; - let history_number = height; + let history_number = height + 1; assert!(self.commit_id_table.get(&confirmed_commit_id)?.is_none()); @@ -130,10 +155,25 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { }); write_schema.write_batch::>(history_indices_table_op); + if let Some(this_min_k) = updates.keys().min().cloned() { + let update_min_k = need_update_min_key(confirmed_min_key.as_ref(), &this_min_k); + if update_min_k { + confirmed_min_key = Some(this_min_k); + } + } + self.change_history_table .commit(history_number, updates.into_iter(), write_schema)?; } + if let Some(confirmed_min_k) = confirmed_min_key { + let previous_min_key = self.history_min_key.read().clone(); + let update_min_k = need_update_min_key(previous_min_key.as_ref(), &confirmed_min_k); + if update_min_k { + *self.history_min_key.write() = Some(confirmed_min_k); + } + } + Ok(()) } } @@ -144,11 +184,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { commit_id: &CommitID, key: &'a T::Key, ) -> Result)>> { - let query_number = if let Some(value) = self.commit_id_table.get(commit_id)? { - value.into_owned() - } else { - return Err(StorageError::CommitIDNotFound); - }; + let query_number = self.get_history_number_by_commit_id(*commit_id)?; let mut num_items = 0; let range_query_key = HistoryIndexKey(key.clone(), query_number); @@ -188,6 +224,23 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { }); Ok(history_iter) } + + fn get_versioned_store_history_part( + &self, + commit_id: &CommitID, + ) -> Result> { + let query_number = self.get_history_number_by_commit_id(*commit_id)?; + let mut key_opt = self.history_min_key.read().as_ref().cloned(); + let mut map = BTreeMap::new(); + while let Some(key) = key_opt { + let value = self.get_historical_part(query_number, &key)?; + if let Some(value) = value { + map.insert(key.clone(), value); + } + key_opt = self.find_larger_key(&key)?; + } + Ok(map) + } } pub struct OneStore { @@ -230,8 +283,9 @@ impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager Ok(OneStore::from_map(pending_map)), Err(PendingError::CommitIDNotFound(target_commit_id)) => { assert_eq!(target_commit_id, *commit); - let history_map: BTreeMap = BTreeMap::new(); //todo - Ok(OneStore::from_map(history_map)) + Ok(OneStore::from_map( + self.get_versioned_store_history_part(commit)?, + )) } Err(other_err) => Err(StorageError::PendingError(other_err)), } From 71617f5208b4a2f41ec9e1dd6d693798ea5dba86 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:32:06 +0800 Subject: [PATCH 70/79] split to key_value_store_manager_impl.rs --- .../key_value_store_manager_impl.rs | 184 ++++++++++++++++++ .../versioned_flat_key_value/mod.rs | 176 +---------------- 2 files changed, 186 insertions(+), 174 deletions(-) create mode 100644 src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs diff --git a/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs b/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs new file mode 100644 index 0000000..895f813 --- /dev/null +++ b/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs @@ -0,0 +1,184 @@ +use std::{collections::BTreeMap, marker::PhantomData}; + +use crate::{ + errors::Result, + middlewares::CommitID, + traits::{KeyValueStore, KeyValueStoreBulksTrait, KeyValueStoreManager}, + StorageError, +}; + +use super::{table_schema::VersionedKeyValueSchema, HistoryIndexKey, PendingError, VersionedStore}; + +pub struct OneStore { + map: BTreeMap, + _marker: PhantomData, +} + +impl OneStore { + pub fn from_map(map: BTreeMap) -> Self { + OneStore { + map, + _marker: PhantomData, + } + } +} + +impl KeyValueStore + for OneStore +{ + fn get(&self, key: &K) -> Result> { + Ok(self.map.get(key).cloned()) + } + + fn iter<'a>(&'a self, key: &K) -> Result> { + Ok(self.map.range(key..)) + } + + fn commit(self, commit: C, changes: impl Iterator) { + todo!() + } +} + +// Trait methods implementation +impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager + for VersionedStore<'db, T> +{ + type Store = OneStore; + fn get_versioned_store(&self, commit: &CommitID) -> Result { + let pending_res = self.pending_part.get_versioned_store(*commit); + match pending_res { + Ok(pending_map) => Ok(OneStore::from_map(pending_map)), + Err(PendingError::CommitIDNotFound(target_commit_id)) => { + assert_eq!(target_commit_id, *commit); + Ok(OneStore::from_map( + self.get_versioned_store_history_part(commit)?, + )) + } + Err(other_err) => Err(StorageError::PendingError(other_err)), + } + } + + #[allow(clippy::type_complexity)] + fn iter_historical_changes<'a>( + &'a self, + commit_id: &CommitID, + key: &'a T::Key, + ) -> Result)>>> { + let pending_res = self.pending_part.iter_historical_changes(commit_id, key); + match pending_res { + Ok(pending_iter) => { + if let Some(history_commit) = self.pending_part.get_parent_of_root() { + let history_iter = + self.iter_historical_changes_history_part(&history_commit, key)?; + Ok(Box::new(pending_iter.chain(history_iter))) + } else { + Ok(Box::new(pending_iter)) + } + } + Err(PendingError::CommitIDNotFound(target_commit)) => { + assert_eq!(target_commit, *commit_id); + let history_iter = + self.iter_historical_changes_history_part(&target_commit, key)?; + Ok(Box::new(history_iter)) + } + Err(other_err) => Err(StorageError::PendingError(other_err)), + } + } + + fn discard(&mut self, commit: CommitID) -> Result<()> { + Ok(self.pending_part.discard(commit)?) + } + + fn get_versioned_key(&self, commit: &CommitID, key: &T::Key) -> Result> { + // let pending_res = self.pending_part.query_frequent_commit_id(commit, key); // this will checkout_current + let pending_res = self.pending_part.get_versioned_key(commit, key); // this does not checkout_current + let history_commit = match pending_res { + Ok(Some(value)) => { + return Ok(value); + } + Ok(None) => { + if let Some(commit) = self.pending_part.get_parent_of_root() { + commit + } else { + return Ok(None); + } + } + Err(PendingError::CommitIDNotFound(target_commit)) => { + assert_eq!(target_commit, *commit); + target_commit + } + Err(other_err) => { + return Err(StorageError::PendingError(other_err)); + } + }; + + let history_number = self.get_history_number_by_commit_id(history_commit)?; + self.get_historical_part(history_number, key) + } +} + +// Helper methods used in trait implementations +impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { + fn iter_historical_changes_history_part<'a>( + &'a self, + commit_id: &CommitID, + key: &'a T::Key, + ) -> Result)>> { + let query_number = self.get_history_number_by_commit_id(*commit_id)?; + + let mut num_items = 0; + let range_query_key = HistoryIndexKey(key.clone(), query_number); + for item in self.history_index_table.iter(&range_query_key)? { + let (k_with_history_number, indices) = item?; + let HistoryIndexKey(k, history_number) = k_with_history_number.as_ref(); + if k != key { + break; + } else { + let found_version_number = indices.as_ref().last(*history_number); + let found_value = self + .change_history_table + .get_versioned_key(&found_version_number, key)?; + let found_commit_id = self.history_number_table.get(&found_version_number)?; + num_items += 1; + } + } + + let history_iter = self + .history_index_table + .iter(&range_query_key)? + .take(num_items) + .map(move |item| { + let (k_with_history_number, indices) = + item.expect("previous for + take() should truncate before err"); + let HistoryIndexKey(k, history_number) = k_with_history_number.as_ref(); + let found_version_number = indices.as_ref().last(*history_number); + let found_value = self + .change_history_table + .get_versioned_key(&found_version_number, key) + .expect("previous for + take() should truncate before err"); + let found_commit_id = self + .history_number_table + .get(&found_version_number) + .expect("previous for + take() should truncate before err"); + (found_commit_id.unwrap().into_owned(), key, found_value) + }); + Ok(history_iter) + } + + fn get_versioned_store_history_part( + &self, + commit_id: &CommitID, + ) -> Result> { + let query_number = self.get_history_number_by_commit_id(*commit_id)?; + let mut key_opt = self.history_min_key.read().as_ref().cloned(); + let mut map = BTreeMap::new(); + while let Some(key) = key_opt { + let value = self.get_historical_part(query_number, &key)?; + if let Some(value) = value { + map.insert(key.clone(), value); + } + key_opt = self.find_larger_key(&key)?; + } + Ok(map) + } +} diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index f3f91a0..a9eb5ca 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -1,10 +1,10 @@ +mod key_value_store_manager_impl; mod pending_part; mod serde; mod table_schema; use std::borrow::Cow; use std::collections::BTreeMap; -use std::marker::PhantomData; use parking_lot::RwLock; pub use pending_part::PendingError; @@ -19,7 +19,7 @@ use super::CommitIDSchema; use crate::backends::{TableReader, WriteSchemaTrait}; use crate::errors::Result; use crate::middlewares::{CommitID, HistoryNumber, KeyValueStoreBulks}; -use crate::traits::{KeyValueStore, KeyValueStoreBulksTrait, KeyValueStoreManager}; +use crate::traits::KeyValueStoreBulksTrait; use crate::StorageError; #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] @@ -177,175 +177,3 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { Ok(()) } } - -impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { - fn iter_historical_changes_history_part<'a>( - &'a self, - commit_id: &CommitID, - key: &'a T::Key, - ) -> Result)>> { - let query_number = self.get_history_number_by_commit_id(*commit_id)?; - - let mut num_items = 0; - let range_query_key = HistoryIndexKey(key.clone(), query_number); - for item in self.history_index_table.iter(&range_query_key)? { - let (k_with_history_number, indices) = item?; - let HistoryIndexKey(k, history_number) = k_with_history_number.as_ref(); - if k != key { - break; - } else { - let found_version_number = indices.as_ref().last(*history_number); - let found_value = self - .change_history_table - .get_versioned_key(&found_version_number, key)?; - let found_commit_id = self.history_number_table.get(&found_version_number)?; - num_items += 1; - } - } - - let history_iter = self - .history_index_table - .iter(&range_query_key)? - .take(num_items) - .map(move |item| { - let (k_with_history_number, indices) = - item.expect("previous for + take() should truncate before err"); - let HistoryIndexKey(k, history_number) = k_with_history_number.as_ref(); - let found_version_number = indices.as_ref().last(*history_number); - let found_value = self - .change_history_table - .get_versioned_key(&found_version_number, key) - .expect("previous for + take() should truncate before err"); - let found_commit_id = self - .history_number_table - .get(&found_version_number) - .expect("previous for + take() should truncate before err"); - (found_commit_id.unwrap().into_owned(), key, found_value) - }); - Ok(history_iter) - } - - fn get_versioned_store_history_part( - &self, - commit_id: &CommitID, - ) -> Result> { - let query_number = self.get_history_number_by_commit_id(*commit_id)?; - let mut key_opt = self.history_min_key.read().as_ref().cloned(); - let mut map = BTreeMap::new(); - while let Some(key) = key_opt { - let value = self.get_historical_part(query_number, &key)?; - if let Some(value) = value { - map.insert(key.clone(), value); - } - key_opt = self.find_larger_key(&key)?; - } - Ok(map) - } -} - -pub struct OneStore { - map: BTreeMap, - _marker: PhantomData, -} - -impl OneStore { - pub fn from_map(map: BTreeMap) -> Self { - OneStore { - map, - _marker: PhantomData, - } - } -} - -impl KeyValueStore - for OneStore -{ - fn get(&self, key: &K) -> Result> { - Ok(self.map.get(key).cloned()) - } - - fn iter<'a>(&'a self, key: &K) -> Result> { - Ok(self.map.range(key..)) - } - - fn commit(self, commit: C, changes: impl Iterator) { - todo!() - } -} - -impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager - for VersionedStore<'db, T> -{ - type Store = OneStore; - fn get_versioned_store(&self, commit: &CommitID) -> Result { - let pending_res = self.pending_part.get_versioned_store(*commit); - match pending_res { - Ok(pending_map) => Ok(OneStore::from_map(pending_map)), - Err(PendingError::CommitIDNotFound(target_commit_id)) => { - assert_eq!(target_commit_id, *commit); - Ok(OneStore::from_map( - self.get_versioned_store_history_part(commit)?, - )) - } - Err(other_err) => Err(StorageError::PendingError(other_err)), - } - } - - #[allow(clippy::type_complexity)] - fn iter_historical_changes<'a>( - &'a self, - commit_id: &CommitID, - key: &'a T::Key, - ) -> Result)>>> { - let pending_res = self.pending_part.iter_historical_changes(commit_id, key); - match pending_res { - Ok(pending_iter) => { - if let Some(history_commit) = self.pending_part.get_parent_of_root() { - let history_iter = - self.iter_historical_changes_history_part(&history_commit, key)?; - Ok(Box::new(pending_iter.chain(history_iter))) - } else { - Ok(Box::new(pending_iter)) - } - } - Err(PendingError::CommitIDNotFound(target_commit)) => { - assert_eq!(target_commit, *commit_id); - let history_iter = - self.iter_historical_changes_history_part(&target_commit, key)?; - Ok(Box::new(history_iter)) - } - Err(other_err) => Err(StorageError::PendingError(other_err)), - } - } - - fn discard(&mut self, commit: CommitID) -> Result<()> { - Ok(self.pending_part.discard(commit)?) - } - - fn get_versioned_key(&self, commit: &CommitID, key: &T::Key) -> Result> { - // let pending_res = self.pending_part.get_versioned_key(commit, key); // this will checkout_current - let pending_res = self.pending_part.get_versioned_key(commit, key); // this does not checkout_current - let history_commit = match pending_res { - Ok(Some(value)) => { - return Ok(value); - } - Ok(None) => { - if let Some(commit) = self.pending_part.get_parent_of_root() { - commit - } else { - return Ok(None); - } - } - Err(PendingError::CommitIDNotFound(target_commit)) => { - assert_eq!(target_commit, *commit); - target_commit - } - Err(other_err) => { - return Err(StorageError::PendingError(other_err)); - } - }; - - let history_number = self.get_history_number_by_commit_id(history_commit)?; - self.get_historical_part(history_number, key) - } -} From e6232d0c7f2ca09516ab4cc22ee60fdff88e2709 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:47:36 +0800 Subject: [PATCH 71/79] rearrange function organization for VersionedStore --- .../key_value_store_manager_impl.rs | 2 +- .../versioned_flat_key_value/mod.rs | 26 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs b/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs index 895f813..8b31457 100644 --- a/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs +++ b/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs @@ -177,7 +177,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { if let Some(value) = value { map.insert(key.clone(), value); } - key_opt = self.find_larger_key(&key)?; + key_opt = self.find_larger_historical_key(&key)?; } Ok(map) } diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index a9eb5ca..0ea2de0 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -46,6 +46,7 @@ pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { history_min_key: RwLock>, } +// private helper methods impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { fn get_history_number_by_commit_id(&self, commit: CommitID) -> Result { if let Some(value) = self.commit_id_table.get(&commit)? { @@ -55,16 +56,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { } } - pub fn add_to_pending_part( - &mut self, - parent_commit: Option, - commit: CommitID, - updates: BTreeMap>, - ) -> Result<()> { - Ok(self.pending_part.add_node(updates, commit, parent_commit)?) - } - - pub fn get_historical_part( + fn get_historical_part( &self, query_version_number: HistoryNumber, key: &T::Key, @@ -92,7 +84,7 @@ impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { .get_versioned_key(&found_version_number, key) } - pub fn find_larger_key(&self, key: &T::Key) -> Result> { + fn find_larger_historical_key(&self, key: &T::Key) -> Result> { // todo: here correct? let range_query_key = HistoryIndexKey(key.clone(), MIN_HISTORY_NUMBER_MINUS_ONE); @@ -115,8 +107,18 @@ fn need_update_min_key(original_min: Option<&K>, challenge_min: &K) -> b } } +// callable methods impl<'db, T: VersionedKeyValueSchema> VersionedStore<'db, T> { - fn confirmed_pending_to_history( + pub fn add_to_pending_part( + &mut self, + parent_commit: Option, + commit: CommitID, + updates: BTreeMap>, + ) -> Result<()> { + Ok(self.pending_part.add_node(updates, commit, parent_commit)?) + } + + pub fn confirmed_pending_to_history( &mut self, new_root_commit_id: CommitID, write_schema: &impl WriteSchemaTrait, From 9df9ce83185b7e16757de6961456f895b16f3041 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:17:41 +0800 Subject: [PATCH 72/79] remove unnecessary funcs --- .../versioned_flat_key_value/pending_part/commit_tree.rs | 4 ---- .../pending_part/versioned_hash_map.rs | 4 ---- 2 files changed, 8 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index df21358..ff01b49 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -41,10 +41,6 @@ impl Tree { } } - pub(super) fn get_height_of_root(&self) -> usize { - self.height_of_root - } - pub(super) fn contains_commit_id(&self, commit_id: &S::CommitId) -> bool { self.index_map.contains_key(commit_id) } diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index ed908e2..435e738 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -84,10 +84,6 @@ impl VersionedHashMap { self.parent_of_root } - pub fn get_height_of_root(&self) -> usize { - self.tree.get_height_of_root() - } - pub fn add_node( &mut self, updates: KeyValueMap, From ddf3c5d96f22a4cb923fb9ca738f7932da43224e Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:24:17 +0800 Subject: [PATCH 73/79] modify tests for pending part's get_versioned_key --- .../pending_part/versioned_hash_map.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs index 435e738..8ecf9bc 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs @@ -356,6 +356,9 @@ mod tests { let commit_id = rng.gen_range(1..=num_nodes) as CommitId; for ikey in 0..10 { let key: u64 = ikey; + let versioned_value = versioned_hash_map + .get_versioned_key(&commit_id, &key) + .unwrap(); let result = versioned_hash_map .query_frequent_commit_id(commit_id, &key) .unwrap(); @@ -363,6 +366,7 @@ mod tests { .get_apply_map_from_root_included(commit_id) .unwrap(); let answer = apply_map.get(&key).map(|a| a.value.clone()); + assert_eq!(versioned_value, answer); assert_eq!(result, answer); } } From e5dda2f1cb55d5251abc881c2fe6803652eaafa4 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:26:58 +0800 Subject: [PATCH 74/79] rename versioned_map --- .../versioned_flat_key_value/mod.rs | 4 +-- .../pending_part/mod.rs | 4 +-- ...versioned_hash_map.rs => versioned_map.rs} | 26 +++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) rename src/middlewares/versioned_flat_key_value/pending_part/{versioned_hash_map.rs => versioned_map.rs} (94%) diff --git a/src/middlewares/versioned_flat_key_value/mod.rs b/src/middlewares/versioned_flat_key_value/mod.rs index 0ea2de0..64b074a 100644 --- a/src/middlewares/versioned_flat_key_value/mod.rs +++ b/src/middlewares/versioned_flat_key_value/mod.rs @@ -11,7 +11,7 @@ pub use pending_part::PendingError; use self::pending_part::pending_schema::PendingKeyValueConfig; use self::table_schema::{HistoryChangeTable, HistoryIndicesTable, VersionedKeyValueSchema}; -use pending_part::VersionedHashMap; +use pending_part::VersionedMap; use super::commit_id_schema::HistoryNumberSchema; use super::ChangeKey; @@ -38,7 +38,7 @@ impl HistoryIndices { const MIN_HISTORY_NUMBER_MINUS_ONE: HistoryNumber = 0; pub struct VersionedStore<'db, T: VersionedKeyValueSchema> { - pending_part: &'db mut VersionedHashMap>, + pending_part: &'db mut VersionedMap>, history_index_table: TableReader<'db, HistoryIndicesTable>, commit_id_table: TableReader<'db, CommitIDSchema>, history_number_table: TableReader<'db, HistoryNumberSchema>, diff --git a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs index 2e154a3..db589be 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs @@ -1,7 +1,7 @@ mod commit_tree; pub mod error; pub mod pending_schema; -pub mod versioned_hash_map; +pub mod versioned_map; pub use error::PendingError; -pub use versioned_hash_map::VersionedHashMap; +pub use versioned_map::VersionedMap; diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs similarity index 94% rename from src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs rename to src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs index 8ecf9bc..15677d6 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_hash_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs @@ -11,7 +11,7 @@ use super::{ use parking_lot::RwLock; -pub struct VersionedHashMap { +pub struct VersionedMap { parent_of_root: Option, tree: Tree, current: RwLock>>, @@ -50,9 +50,9 @@ impl CurrentMap { } } -impl VersionedHashMap { +impl VersionedMap { pub fn new(parent_of_root: Option, height_of_root: usize) -> Self { - VersionedHashMap { + VersionedMap { parent_of_root, tree: Tree::new(height_of_root), current: RwLock::new(None), @@ -60,7 +60,7 @@ impl VersionedHashMap { } } -impl VersionedHashMap { +impl VersionedMap { #[allow(clippy::type_complexity)] // root..=new_root's parent: (commit_id, key_value_map) pub fn change_root( @@ -79,7 +79,7 @@ impl VersionedHashMap { } } -impl VersionedHashMap { +impl VersionedMap { pub fn get_parent_of_root(&self) -> Option { self.parent_of_root } @@ -153,7 +153,7 @@ impl VersionedHashMap { } } -impl VersionedHashMap { +impl VersionedMap { // None: pending_part not know // Some(None): pending_part know that this key has been deleted // Some(Some(value)): pending_part know this key's value @@ -178,7 +178,7 @@ impl VersionedHashMap { } // only one key, no need to invoke checkout_current -impl VersionedHashMap { +impl VersionedMap { pub fn iter_historical_changes<'a>( &'a self, commit_id: &S::CommitId, @@ -225,7 +225,7 @@ impl VersionedHashMap { } } -impl VersionedHashMap { +impl VersionedMap { fn checkout_current(&self, target_commit_id: S::CommitId) -> PendResult<(), S> { let current_commit_id = self.current.read().as_ref().map(|c| c.commit_id); if let Some(current_commit_id) = current_commit_id { @@ -299,9 +299,9 @@ mod tests { fn generate_random_tree( num_nodes: usize, rng: &mut StdRng, - ) -> (Tree, VersionedHashMap) { + ) -> (Tree, VersionedMap) { let mut forward_only_tree = Tree::new(0); - let mut versioned_hash_map = VersionedHashMap::new(None, 0); + let mut versioned_hash_map = VersionedMap::new(None, 0); for i in 1..=num_nodes as CommitId { let parent_commit_id = if i == 1 { @@ -375,7 +375,7 @@ mod tests { #[test] fn test_multiple_roots_err() { let mut forward_only_tree = Tree::::new(0); - let mut versioned_hash_map = VersionedHashMap::::new(None, 0); + let mut versioned_hash_map = VersionedMap::::new(None, 0); forward_only_tree.add_root(0, BTreeMap::new()).unwrap(); versioned_hash_map @@ -395,7 +395,7 @@ mod tests { #[test] fn test_commit_id_not_found_err() { let mut forward_only_tree = Tree::::new(0); - let mut versioned_hash_map = VersionedHashMap::::new(None, 0); + let mut versioned_hash_map = VersionedMap::::new(None, 0); assert_eq!( forward_only_tree.add_non_root_node(1, 0, BTreeMap::new()), @@ -410,7 +410,7 @@ mod tests { #[test] fn test_commit_id_already_exists_err() { let mut forward_only_tree = Tree::::new(0); - let mut versioned_hash_map = VersionedHashMap::::new(None, 0); + let mut versioned_hash_map = VersionedMap::::new(None, 0); forward_only_tree.add_root(0, BTreeMap::new()).unwrap(); versioned_hash_map From db1275d583b5e3097f35ade1589a108af52620f8 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:40:50 +0800 Subject: [PATCH 75/79] split current_map.rs --- .../pending_part/current_map.rs | 48 +++++++++++++++++ .../pending_part/mod.rs | 1 + .../pending_part/versioned_map.rs | 53 ++++--------------- 3 files changed, 58 insertions(+), 44 deletions(-) create mode 100644 src/middlewares/versioned_flat_key_value/pending_part/current_map.rs diff --git a/src/middlewares/versioned_flat_key_value/pending_part/current_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/current_map.rs new file mode 100644 index 0000000..2755e9d --- /dev/null +++ b/src/middlewares/versioned_flat_key_value/pending_part/current_map.rs @@ -0,0 +1,48 @@ +use std::collections::BTreeMap; + +use super::pending_schema::{ApplyMap, ApplyRecord, PendingKeyValueSchema}; + +pub struct CurrentMap { + map: BTreeMap>, + commit_id: S::CommitId, +} + +impl CurrentMap { + pub fn new(commit_id: S::CommitId) -> Self { + Self { + map: BTreeMap::new(), + commit_id, + } + } + + pub fn get_map(&self) -> &BTreeMap> { + &self.map + } + + pub fn get_commit_id(&self) -> &S::CommitId { + &self.commit_id + } + + pub fn set_commit_id(&mut self, commit_id: S::CommitId) { + self.commit_id = commit_id; + } + + pub fn rollback(&mut self, rollbacks: BTreeMap>>) { + for (key, to_rollback) in rollbacks.into_iter() { + match to_rollback { + None => { + self.map.remove(&key); + } + Some(to_rollback_record) => { + self.map.insert(key, to_rollback_record); + } + } + } + } + + pub fn apply(&mut self, applys: ApplyMap) { + for (key, apply) in applys.into_iter() { + self.map.insert(key, apply); + } + } +} diff --git a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs index db589be..7d5eb81 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs @@ -1,4 +1,5 @@ mod commit_tree; +mod current_map; pub mod error; pub mod pending_schema; pub mod versioned_map; diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs index 15677d6..d69ee99 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs @@ -2,10 +2,8 @@ use std::collections::BTreeMap; use super::{ commit_tree::Tree, - pending_schema::{ - ApplyMap, ApplyRecord, KeyValueMap, PendingKeyValueSchema, RecoverRecord, - Result as PendResult, - }, + current_map::CurrentMap, + pending_schema::{KeyValueMap, PendingKeyValueSchema, RecoverRecord, Result as PendResult}, PendingError, }; @@ -17,39 +15,6 @@ pub struct VersionedMap { current: RwLock>>, } -struct CurrentMap { - map: BTreeMap>, - commit_id: S::CommitId, -} - -impl CurrentMap { - fn new(commit_id: S::CommitId) -> Self { - Self { - map: BTreeMap::new(), - commit_id, - } - } - - fn rollback(&mut self, rollbacks: BTreeMap>>) { - for (key, to_rollback) in rollbacks.into_iter() { - match to_rollback { - None => { - self.map.remove(&key); - } - Some(to_rollback_record) => { - self.map.insert(key, to_rollback_record); - } - } - } - } - - fn apply(&mut self, applys: ApplyMap) { - for (key, apply) in applys.into_iter() { - self.map.insert(key, apply); - } - } -} - impl VersionedMap { pub fn new(parent_of_root: Option, height_of_root: usize) -> Self { VersionedMap { @@ -137,7 +102,7 @@ impl VersionedMap { let current = current_read.as_ref().unwrap(); let mut modifications = BTreeMap::new(); for (key, value) in updates.into_iter() { - let last_commit_id = current.map.get(&key).map(|s| s.commit_id); + let last_commit_id = current.get_map().get(&key).map(|s| s.commit_id); modifications.insert( key, RecoverRecord { @@ -171,7 +136,7 @@ impl VersionedMap { .read() .as_ref() .unwrap() - .map + .get_map() .get(key) .map(|c| c.value.clone())) } @@ -197,7 +162,7 @@ impl VersionedMap { pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult<(), S> { self.tree.discard(commit_id)?; - let current_commit_id = self.current.read().as_ref().map(|c| c.commit_id); + let current_commit_id = self.current.read().as_ref().map(|c| *c.get_commit_id()); if let Some(current_commit_id) = current_commit_id { if !self.tree.contains_commit_id(¤t_commit_id) { *self.current.write() = None; @@ -215,7 +180,7 @@ impl VersionedMap { let map: BTreeMap<_, _> = current_read .as_ref() .unwrap() - .map + .get_map() .iter() .filter_map(|(k, apply_record)| { apply_record.value.as_ref().map(|v| (k.clone(), v.clone())) @@ -227,7 +192,7 @@ impl VersionedMap { impl VersionedMap { fn checkout_current(&self, target_commit_id: S::CommitId) -> PendResult<(), S> { - let current_commit_id = self.current.read().as_ref().map(|c| c.commit_id); + let current_commit_id = self.current.read().as_ref().map(|c| *c.get_commit_id()); if let Some(current_commit_id) = current_commit_id { let (rollbacks, applys) = self .tree @@ -236,7 +201,7 @@ impl VersionedMap { let current = current_option.as_mut().unwrap(); current.rollback(rollbacks); current.apply(applys); - current.commit_id = target_commit_id; + current.set_commit_id(target_commit_id); } else { let mut current = CurrentMap::::new(target_commit_id); let applys = self @@ -247,7 +212,7 @@ impl VersionedMap { } assert_eq!( - self.current.read().as_ref().unwrap().commit_id, + *self.current.read().as_ref().unwrap().get_commit_id(), target_commit_id ); From 402ed48e40e4821da6dcbd08126ff6e11dd579bd Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:08:30 +0800 Subject: [PATCH 76/79] rearrange versioned_map's order of funcs --- .../key_value_store_manager_impl.rs | 4 +- .../pending_part/commit_tree.rs | 1 - .../pending_part/versioned_map.rs | 138 +++++++++--------- 3 files changed, 72 insertions(+), 71 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs b/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs index 8b31457..0a13ec8 100644 --- a/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs +++ b/src/middlewares/versioned_flat_key_value/key_value_store_manager_impl.rs @@ -90,8 +90,8 @@ impl<'db, T: VersionedKeyValueSchema> KeyValueStoreManager Result> { - // let pending_res = self.pending_part.query_frequent_commit_id(commit, key); // this will checkout_current - let pending_res = self.pending_part.get_versioned_key(commit, key); // this does not checkout_current + // let pending_res = self.pending_part.get_versioned_key_with_checkout(commit, key); // this will checkout_current + let pending_res = self.pending_part.get_versioned_key(commit, key); let history_commit = match pending_res { Ok(Some(value)) => { return Ok(value); diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs index ff01b49..60d96f8 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs @@ -16,7 +16,6 @@ pub struct TreeNode { // todo: test lazy height // height will not be changed even when root is changed - // height is only used for lca in Tree.collect_rollback_and_apply_ops() height: usize, commit_id: S::CommitId, diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs index d69ee99..39e5626 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs @@ -23,32 +23,43 @@ impl VersionedMap { current: RwLock::new(None), } } + + pub fn get_parent_of_root(&self) -> Option { + self.parent_of_root + } } impl VersionedMap { - #[allow(clippy::type_complexity)] - // root..=new_root's parent: (commit_id, key_value_map) - pub fn change_root( - &mut self, - commit_id: S::CommitId, - ) -> PendResult<(usize, Vec<(S::CommitId, KeyValueMap)>), S> { - // to_commit: root..=new_root's parent - let (start_height_to_commit, to_commit) = self.tree.change_root(commit_id)?; - - if let Some(parent_of_new_root) = to_commit.last() { - self.parent_of_root = Some(parent_of_new_root.0); - *self.current.write() = None; + fn checkout_current(&self, target_commit_id: S::CommitId) -> PendResult<(), S> { + let current_commit_id = self.current.read().as_ref().map(|c| *c.get_commit_id()); + if let Some(current_commit_id) = current_commit_id { + let (rollbacks, applys) = self + .tree + .collect_rollback_and_apply_ops(current_commit_id, target_commit_id)?; + let mut current_option = self.current.write(); + let current = current_option.as_mut().unwrap(); + current.rollback(rollbacks); + current.apply(applys); + current.set_commit_id(target_commit_id); + } else { + let mut current = CurrentMap::::new(target_commit_id); + let applys = self + .tree + .get_apply_map_from_root_included(target_commit_id)?; + current.apply(applys); + *self.current.write() = Some(current); } - Ok((start_height_to_commit, to_commit)) + assert_eq!( + *self.current.read().as_ref().unwrap().get_commit_id(), + target_commit_id + ); + + Ok(()) } } impl VersionedMap { - pub fn get_parent_of_root(&self) -> Option { - self.parent_of_root - } - pub fn add_node( &mut self, updates: KeyValueMap, @@ -119,30 +130,26 @@ impl VersionedMap { } impl VersionedMap { - // None: pending_part not know - // Some(None): pending_part know that this key has been deleted - // Some(Some(value)): pending_part know this key's value - pub fn query_frequent_commit_id( - &self, + #[allow(clippy::type_complexity)] + // root..=new_root's parent: (commit_id, key_value_map) + pub fn change_root( + &mut self, commit_id: S::CommitId, - key: &S::Key, - ) -> PendResult>, S> { - // let query node to be self.current - self.checkout_current(commit_id)?; + ) -> PendResult<(usize, Vec<(S::CommitId, KeyValueMap)>), S> { + // to_commit: root..=new_root's parent + let (start_height_to_commit, to_commit) = self.tree.change_root(commit_id)?; - // query - Ok(self - .current - .read() - .as_ref() - .unwrap() - .get_map() - .get(key) - .map(|c| c.value.clone())) + if let Some(parent_of_new_root) = to_commit.last() { + self.parent_of_root = Some(parent_of_new_root.0); + *self.current.write() = None; + } + + Ok((start_height_to_commit, to_commit)) } } -// only one key, no need to invoke checkout_current +// Helper methods in pending part to support +// impl KeyValueStoreManager for VersionedStore impl VersionedMap { pub fn iter_historical_changes<'a>( &'a self, @@ -152,6 +159,9 @@ impl VersionedMap { self.tree.iter_historical_changes(commit_id, key) } + // None: pending_part not know + // Some(None): pending_part know that this key has been deleted + // Some(Some(value)): pending_part know this key's value pub fn get_versioned_key( &self, commit_id: &S::CommitId, @@ -160,6 +170,28 @@ impl VersionedMap { self.tree.get_versioned_key(commit_id, key) } + // alternative method of self.get_versioned_key(), + // but it invokes self.checkout_current(), + // thus is only suitable for frequent commit_id + pub fn get_versioned_key_with_checkout( + &self, + commit_id: S::CommitId, + key: &S::Key, + ) -> PendResult>, S> { + // let query node to be self.current + self.checkout_current(commit_id)?; + + // query + Ok(self + .current + .read() + .as_ref() + .unwrap() + .get_map() + .get(key) + .map(|c| c.value.clone())) + } + pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult<(), S> { self.tree.discard(commit_id)?; let current_commit_id = self.current.read().as_ref().map(|c| *c.get_commit_id()); @@ -190,36 +222,6 @@ impl VersionedMap { } } -impl VersionedMap { - fn checkout_current(&self, target_commit_id: S::CommitId) -> PendResult<(), S> { - let current_commit_id = self.current.read().as_ref().map(|c| *c.get_commit_id()); - if let Some(current_commit_id) = current_commit_id { - let (rollbacks, applys) = self - .tree - .collect_rollback_and_apply_ops(current_commit_id, target_commit_id)?; - let mut current_option = self.current.write(); - let current = current_option.as_mut().unwrap(); - current.rollback(rollbacks); - current.apply(applys); - current.set_commit_id(target_commit_id); - } else { - let mut current = CurrentMap::::new(target_commit_id); - let applys = self - .tree - .get_apply_map_from_root_included(target_commit_id)?; - current.apply(applys); - *self.current.write() = Some(current); - } - - assert_eq!( - *self.current.read().as_ref().unwrap().get_commit_id(), - target_commit_id - ); - - Ok(()) - } -} - #[cfg(test)] mod tests { use crate::{ @@ -307,7 +309,7 @@ mod tests { } #[test] - fn test_query() { + fn test_get_versioned_key() { let num_nodes = 100; let seed: [u8; 32] = [ @@ -325,7 +327,7 @@ mod tests { .get_versioned_key(&commit_id, &key) .unwrap(); let result = versioned_hash_map - .query_frequent_commit_id(commit_id, &key) + .get_versioned_key_with_checkout(commit_id, &key) .unwrap(); let apply_map = forward_only_tree .get_apply_map_from_root_included(commit_id) From 341377c12fb7f49d0d85e79d6006f2429453acb2 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:22:35 +0800 Subject: [PATCH 77/79] split tree and node --- .../pending_part/current_map.rs | 4 +- .../pending_part/mod.rs | 2 +- .../{commit_tree.rs => tree/mod.rs} | 132 +++++------------- .../pending_part/tree/node.rs | 106 ++++++++++++++ .../pending_part/versioned_map.rs | 8 +- 5 files changed, 151 insertions(+), 101 deletions(-) rename src/middlewares/versioned_flat_key_value/pending_part/{commit_tree.rs => tree/mod.rs} (73%) create mode 100644 src/middlewares/versioned_flat_key_value/pending_part/tree/node.rs diff --git a/src/middlewares/versioned_flat_key_value/pending_part/current_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/current_map.rs index 2755e9d..cac538f 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/current_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/current_map.rs @@ -19,8 +19,8 @@ impl CurrentMap { &self.map } - pub fn get_commit_id(&self) -> &S::CommitId { - &self.commit_id + pub fn get_commit_id(&self) -> S::CommitId { + self.commit_id } pub fn set_commit_id(&mut self, commit_id: S::CommitId) { diff --git a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs index 7d5eb81..3fc598b 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/mod.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/mod.rs @@ -1,7 +1,7 @@ -mod commit_tree; mod current_map; pub mod error; pub mod pending_schema; +mod tree; pub mod versioned_map; pub use error::PendingError; diff --git a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs b/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs similarity index 73% rename from src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs rename to src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs index 60d96f8..afc199e 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/commit_tree.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs @@ -1,30 +1,18 @@ +mod node; + +pub type SlabIndex = usize; + use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use slab::Slab; +use self::node::TreeNode; + use super::pending_schema::{ - ApplyMap, ApplyRecord, KeyValueMap, PendingKeyValueSchema, RecoverMap, RecoverRecord, - Result as PendResult, + ApplyMap, ApplyRecord, KeyValueMap, PendingKeyValueSchema, RecoverMap, Result as PendResult, }; use super::PendingError; -type SlabIndex = usize; - -pub struct TreeNode { - parent: Option, - children: BTreeSet, - - // todo: test lazy height - // height will not be changed even when root is changed - height: usize, - - commit_id: S::CommitId, - // before current node, the old value of this key is modified by which commit_id, - // if none, this key is absent before current node - // here must use CommitID instead of SlabIndex (which may be reused, see slab doc) - modifications: RecoverMap, -} - pub struct Tree { height_of_root: usize, nodes: Slab>, @@ -70,7 +58,7 @@ impl Tree { } fn get_parent_node(&self, node: &TreeNode) -> Option<&TreeNode> { - node.parent + node.get_parent() .map(|p_slab_index| self.get_node_by_slab_index(p_slab_index)) } @@ -80,7 +68,7 @@ impl Tree { key: &S::Key, ) -> PendResult>, S> { let node = self.get_node_by_commit_id(commit_id)?; - Ok(node.modifications.get(key).map(|v| v.value.clone())) + Ok(node.get_modified_value(key)) } } @@ -121,17 +109,17 @@ impl Tree { } // new node - let node = TreeNode::new( + let node = TreeNode::new_non_root( commit_id, parent_slab_index, - self.get_node_by_slab_index(parent_slab_index).height + 1, + self.get_node_by_slab_index(parent_slab_index).get_height() + 1, modifications, ); // add node to tree let slab_index = self.nodes.insert(node); self.index_map.insert(commit_id, slab_index); - self.nodes[parent_slab_index].children.insert(slab_index); + self.nodes[parent_slab_index].insert_child(slab_index); Ok(()) } @@ -145,7 +133,7 @@ impl Tree { while head < slab_indices.len() { let node = self.get_node_by_slab_index(slab_indices[head]); - for &child_index in &node.children { + for &child_index in node.get_children() { slab_indices.push(child_index); } @@ -159,9 +147,9 @@ impl Tree { fn find_path(&self, target_slab_index: SlabIndex) -> Vec<(S::CommitId, KeyValueMap)> { let mut target_node = self.get_node_by_slab_index(target_slab_index); let mut path = VecDeque::new(); - while let Some(parent_slab_index) = target_node.parent { + while let Some(parent_slab_index) = target_node.get_parent() { target_node = self.get_node_by_slab_index(parent_slab_index); - path.push_front((target_node.commit_id, target_node.get_updates())); + path.push_front((target_node.get_commit_id(), target_node.get_updates())); } path.into() } @@ -179,7 +167,10 @@ impl Tree { // early return if new_root == root if to_commit.is_empty() { - return Ok((self.nodes[slab_index].height - to_commit.len(), to_commit)); + return Ok(( + self.nodes[slab_index].get_height() - to_commit.len(), + to_commit, + )); } // subtree of new_root @@ -194,14 +185,18 @@ impl Tree { } } for idx in to_remove { - self.index_map.remove(&self.nodes.remove(idx).commit_id); + self.index_map + .remove(&self.nodes.remove(idx).get_commit_id()); } // set new_root's parent as None - self.nodes[slab_index].parent = None; + self.nodes[slab_index].set_parent(None); // (height of new_root's parent, root..=new_root's parent) - Ok((self.nodes[slab_index].height - to_commit.len(), to_commit)) + Ok(( + self.nodes[slab_index].get_height() - to_commit.len(), + to_commit, + )) } } @@ -214,8 +209,8 @@ impl Tree { let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); let mut path = Vec::new(); while let Some(node) = node_option { - if let Some(RecoverRecord { value, .. }) = node.modifications.get(key) { - path.push((node.commit_id, key, value.clone())); + if let Some(value) = node.get_modified_value(key) { + path.push((node.get_commit_id(), key, value)); } node_option = self.get_parent_node(node); } @@ -229,8 +224,8 @@ impl Tree { ) -> PendResult>, S> { let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); while let Some(node) = node_option { - if let Some(RecoverRecord { value, .. }) = node.modifications.get(key) { - return Ok(Some(value.clone())); + if let Some(value) = node.get_modified_value(key) { + return Ok(Some(value)); } node_option = self.get_parent_node(node); } @@ -241,13 +236,14 @@ impl Tree { pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult<(), S> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; let node = self.get_node_by_slab_index(slab_index); - if let Some(parent_of_discard) = node.parent { + if let Some(parent_of_discard) = node.get_parent() { let to_remove = self.bfs_subtree(slab_index); for idx in to_remove { - self.index_map.remove(&self.nodes.remove(idx).commit_id); + self.index_map + .remove(&self.nodes.remove(idx).get_commit_id()); } let parent_node = self.get_mut_node_by_slab_index(parent_of_discard); - parent_node.children.remove(&slab_index); + parent_node.remove_child(&slab_index); Ok(()) } else { Err(PendingError::RootShouldNotBeDiscarded) @@ -263,7 +259,7 @@ impl Tree { let mut target_node = self.get_node_by_commit_id(target_commit_id)?; let mut commits_rev = BTreeMap::new(); target_node.export_commit_data(&mut commits_rev); - while let Some(parent_slab_index) = target_node.parent { + while let Some(parent_slab_index) = target_node.get_parent() { target_node = self.get_node_by_slab_index(parent_slab_index); target_node.export_commit_data(&mut commits_rev); } @@ -281,15 +277,15 @@ impl Tree { let mut target_node = self.get_node_by_commit_id(target_commit_id)?; let mut rollbacks = BTreeMap::new(); let mut commits_rev = BTreeMap::new(); - while current_node.height > target_node.height { + while current_node.get_height() > target_node.get_height() { current_node.export_rollback_data(&mut rollbacks); current_node = self.get_parent_node(current_node).unwrap(); } - while target_node.height > current_node.height { + while target_node.get_height() > current_node.get_height() { target_node.export_commit_data(&mut commits_rev); target_node = self.get_parent_node(target_node).unwrap(); } - while current_node.commit_id != target_node.commit_id { + while current_node.get_commit_id() != target_node.get_commit_id() { current_node.export_rollback_data(&mut rollbacks); current_node = self.get_parent_node(current_node).unwrap(); target_node.export_commit_data(&mut commits_rev); @@ -329,55 +325,3 @@ impl Tree { Ok((rollbacks_with_value, commits_rev)) } } - -impl TreeNode { - fn new_root(commit_id: S::CommitId, modifications: RecoverMap, height: usize) -> Self { - Self { - height, - commit_id, - parent: None, - children: BTreeSet::new(), - modifications, - } - } - - fn new( - commit_id: S::CommitId, - parent: SlabIndex, - height: usize, - modifications: RecoverMap, - ) -> Self { - Self { - height, - commit_id, - parent: Some(parent), - children: BTreeSet::new(), - modifications, - } - } - - fn get_updates(&self) -> KeyValueMap { - self.modifications - .iter() - .map(|(k, RecoverRecord { value, .. })| (k.clone(), value.clone())) - .collect() - } - - fn export_rollback_data(&self, rollbacks: &mut BTreeMap>) { - for (key, RecoverRecord { last_commit_id, .. }) in self.modifications.iter() { - rollbacks.insert(key.clone(), *last_commit_id); - } - } - - fn export_commit_data(&self, commits_rev: &mut ApplyMap) { - let commit_id = self.commit_id; - for (key, RecoverRecord { value, .. }) in self.modifications.iter() { - commits_rev - .entry(key.clone()) - .or_insert_with(|| ApplyRecord { - commit_id, - value: value.clone(), - }); - } - } -} diff --git a/src/middlewares/versioned_flat_key_value/pending_part/tree/node.rs b/src/middlewares/versioned_flat_key_value/pending_part/tree/node.rs new file mode 100644 index 0000000..a300210 --- /dev/null +++ b/src/middlewares/versioned_flat_key_value/pending_part/tree/node.rs @@ -0,0 +1,106 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::middlewares::versioned_flat_key_value::pending_part::pending_schema::{ + ApplyMap, ApplyRecord, KeyValueMap, PendingKeyValueSchema, RecoverMap, RecoverRecord, +}; + +use super::SlabIndex; + +pub struct TreeNode { + parent: Option, + children: BTreeSet, + + // todo: test lazy height + // height will not be changed even when root is changed + height: usize, + + commit_id: S::CommitId, + // before current node, the old value of this key is modified by which commit_id, + // if none, this key is absent before current node + // here must use CommitID instead of SlabIndex (which may be reused, see slab doc) + modifications: RecoverMap, +} + +impl TreeNode { + pub fn new_root(commit_id: S::CommitId, modifications: RecoverMap, height: usize) -> Self { + Self { + height, + commit_id, + parent: None, + children: BTreeSet::new(), + modifications, + } + } + + pub fn new_non_root( + commit_id: S::CommitId, + parent: SlabIndex, + height: usize, + modifications: RecoverMap, + ) -> Self { + Self { + height, + commit_id, + parent: Some(parent), + children: BTreeSet::new(), + modifications, + } + } + + pub fn get_parent(&self) -> Option { + self.parent + } + + pub fn set_parent(&mut self, new_parent: Option) { + self.parent = new_parent; + } + + pub fn get_children(&self) -> &BTreeSet { + &self.children + } + + pub fn insert_child(&mut self, new_child: SlabIndex) { + self.children.insert(new_child); + } + + pub fn remove_child(&mut self, child_to_remove: &SlabIndex) { + self.children.remove(child_to_remove); + } + + pub fn get_height(&self) -> usize { + self.height + } + + pub fn get_commit_id(&self) -> S::CommitId { + self.commit_id + } + + pub fn get_modified_value(&self, key: &S::Key) -> Option> { + self.modifications.get(key).map(|v| v.value.clone()) + } + + pub fn get_updates(&self) -> KeyValueMap { + self.modifications + .iter() + .map(|(k, RecoverRecord { value, .. })| (k.clone(), value.clone())) + .collect() + } + + pub fn export_rollback_data(&self, rollbacks: &mut BTreeMap>) { + for (key, RecoverRecord { last_commit_id, .. }) in self.modifications.iter() { + rollbacks.insert(key.clone(), *last_commit_id); + } + } + + pub fn export_commit_data(&self, commits_rev: &mut ApplyMap) { + let commit_id = self.commit_id; + for (key, RecoverRecord { value, .. }) in self.modifications.iter() { + commits_rev + .entry(key.clone()) + .or_insert_with(|| ApplyRecord { + commit_id, + value: value.clone(), + }); + } + } +} diff --git a/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs b/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs index 39e5626..9bb5f05 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/versioned_map.rs @@ -1,9 +1,9 @@ use std::collections::BTreeMap; use super::{ - commit_tree::Tree, current_map::CurrentMap, pending_schema::{KeyValueMap, PendingKeyValueSchema, RecoverRecord, Result as PendResult}, + tree::Tree, PendingError, }; @@ -31,7 +31,7 @@ impl VersionedMap { impl VersionedMap { fn checkout_current(&self, target_commit_id: S::CommitId) -> PendResult<(), S> { - let current_commit_id = self.current.read().as_ref().map(|c| *c.get_commit_id()); + let current_commit_id = self.current.read().as_ref().map(|c| c.get_commit_id()); if let Some(current_commit_id) = current_commit_id { let (rollbacks, applys) = self .tree @@ -51,7 +51,7 @@ impl VersionedMap { } assert_eq!( - *self.current.read().as_ref().unwrap().get_commit_id(), + self.current.read().as_ref().unwrap().get_commit_id(), target_commit_id ); @@ -194,7 +194,7 @@ impl VersionedMap { pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult<(), S> { self.tree.discard(commit_id)?; - let current_commit_id = self.current.read().as_ref().map(|c| *c.get_commit_id()); + let current_commit_id = self.current.read().as_ref().map(|c| c.get_commit_id()); if let Some(current_commit_id) = current_commit_id { if !self.tree.contains_commit_id(¤t_commit_id) { *self.current.write() = None; From d229bc5cd09689738bc269b82dabff49a7f49778 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:21:04 +0800 Subject: [PATCH 78/79] rearrange tree's order of funcs --- .../pending_part/tree/mod.rs | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs b/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs index afc199e..a2ad8ca 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs @@ -19,6 +19,7 @@ pub struct Tree { index_map: HashMap, } +// basic methods impl Tree { pub fn new(height_of_root: usize) -> Self { Tree { @@ -70,8 +71,26 @@ impl Tree { let node = self.get_node_by_commit_id(commit_id)?; Ok(node.get_modified_value(key)) } + + // including subroot + fn bfs_subtree(&self, subroot_slab_index: SlabIndex) -> Vec { + let mut slab_indices = vec![subroot_slab_index]; + let mut head = 0; + while head < slab_indices.len() { + let node = self.get_node_by_slab_index(slab_indices[head]); + + for &child_index in node.get_children() { + slab_indices.push(child_index); + } + + head += 1; + } + + slab_indices + } } +// methods to support VersionedMap::add_node() impl Tree { pub fn add_root( &mut self, @@ -125,24 +144,8 @@ impl Tree { } } +// methods to support VersionedMap::change_root() impl Tree { - // including subroot - fn bfs_subtree(&self, subroot_slab_index: SlabIndex) -> Vec { - let mut slab_indices = vec![subroot_slab_index]; - let mut head = 0; - while head < slab_indices.len() { - let node = self.get_node_by_slab_index(slab_indices[head]); - - for &child_index in node.get_children() { - slab_indices.push(child_index); - } - - head += 1; - } - - slab_indices - } - // excluding target fn find_path(&self, target_slab_index: SlabIndex) -> Vec<(S::CommitId, KeyValueMap)> { let mut target_node = self.get_node_by_slab_index(target_slab_index); @@ -200,6 +203,9 @@ impl Tree { } } +// Internal Tree methods supporting +// helper methods in VersionedMap for +// implementing `KeyValueStoreManager` for `VersionedStore`. impl Tree { pub fn iter_historical_changes<'a>( &'a self, @@ -232,7 +238,6 @@ impl Tree { Ok(None) } - // true: is_root, false: is_not_root pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult<(), S> { let slab_index = self.get_slab_index_by_commit_id(commit_id)?; let node = self.get_node_by_slab_index(slab_index); @@ -251,6 +256,7 @@ impl Tree { } } +// methods to support VersionedMap::checkout_current() impl Tree { pub fn get_apply_map_from_root_included( &self, From 1eafd69ad235886aa47464a6f637c515469774c8 Mon Sep 17 00:00:00 2001 From: Rong Ma <160457111+rongma7@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:34:49 +0800 Subject: [PATCH 79/79] split tree into smaller .rs files --- .../pending_part/tree/add_node_support.rs | 62 +++++ .../pending_part/tree/change_root_support.rs | 66 +++++ .../pending_part/tree/checkout_support.rs | 83 ++++++ .../tree/key_value_store_manager_support.rs | 61 +++++ .../pending_part/tree/mod.rs | 252 +----------------- 5 files changed, 278 insertions(+), 246 deletions(-) create mode 100644 src/middlewares/versioned_flat_key_value/pending_part/tree/add_node_support.rs create mode 100644 src/middlewares/versioned_flat_key_value/pending_part/tree/change_root_support.rs create mode 100644 src/middlewares/versioned_flat_key_value/pending_part/tree/checkout_support.rs create mode 100644 src/middlewares/versioned_flat_key_value/pending_part/tree/key_value_store_manager_support.rs diff --git a/src/middlewares/versioned_flat_key_value/pending_part/tree/add_node_support.rs b/src/middlewares/versioned_flat_key_value/pending_part/tree/add_node_support.rs new file mode 100644 index 0000000..bc2f130 --- /dev/null +++ b/src/middlewares/versioned_flat_key_value/pending_part/tree/add_node_support.rs @@ -0,0 +1,62 @@ +use crate::middlewares::{ + versioned_flat_key_value::pending_part::pending_schema::{ + PendingKeyValueSchema, RecoverMap, Result as PendResult, + }, + PendingError, +}; + +use super::{node::TreeNode, Tree}; + +// methods to support VersionedMap::add_node() +impl Tree { + pub fn add_root( + &mut self, + commit_id: S::CommitId, + modifications: RecoverMap, + ) -> PendResult<(), S> { + // return error if there is root + if self.has_root() { + return Err(PendingError::MultipleRootsNotAllowed); + } + // PendingError::CommitIdAlreadyExists(_) cannot happend because no root indicates no node + + // new root + let root = TreeNode::new_root(commit_id, modifications, self.height_of_root); + + // add root to tree + let slab_index = self.nodes.insert(root); + self.index_map.insert(commit_id, slab_index); + + Ok(()) + } + + pub fn add_non_root_node( + &mut self, + commit_id: S::CommitId, + parent_commit_id: S::CommitId, + modifications: RecoverMap, + ) -> PendResult<(), S> { + // return error if parent_commit_id does not exist + let parent_slab_index = self.get_slab_index_by_commit_id(parent_commit_id)?; + + // return error if commit_id exists + if self.contains_commit_id(&commit_id) { + return Err(PendingError::CommitIdAlreadyExists(commit_id)); + } + + // new node + let node = TreeNode::new_non_root( + commit_id, + parent_slab_index, + self.get_node_by_slab_index(parent_slab_index).get_height() + 1, + modifications, + ); + + // add node to tree + let slab_index = self.nodes.insert(node); + self.index_map.insert(commit_id, slab_index); + self.nodes[parent_slab_index].insert_child(slab_index); + + Ok(()) + } +} diff --git a/src/middlewares/versioned_flat_key_value/pending_part/tree/change_root_support.rs b/src/middlewares/versioned_flat_key_value/pending_part/tree/change_root_support.rs new file mode 100644 index 0000000..267760e --- /dev/null +++ b/src/middlewares/versioned_flat_key_value/pending_part/tree/change_root_support.rs @@ -0,0 +1,66 @@ +use std::collections::{BTreeSet, VecDeque}; + +use crate::middlewares::versioned_flat_key_value::pending_part::pending_schema::{ + KeyValueMap, PendingKeyValueSchema, Result as PendResult, +}; + +use super::{SlabIndex, Tree}; + +// methods to support VersionedMap::change_root() +impl Tree { + // excluding target + fn find_path(&self, target_slab_index: SlabIndex) -> Vec<(S::CommitId, KeyValueMap)> { + let mut target_node = self.get_node_by_slab_index(target_slab_index); + let mut path = VecDeque::new(); + while let Some(parent_slab_index) = target_node.get_parent() { + target_node = self.get_node_by_slab_index(parent_slab_index); + path.push_front((target_node.get_commit_id(), target_node.get_updates())); + } + path.into() + } + + // todo: test + #[allow(clippy::type_complexity)] + pub fn change_root( + &mut self, + commit_id: S::CommitId, + ) -> PendResult<(usize, Vec<(S::CommitId, KeyValueMap)>), S> { + let slab_index = self.get_slab_index_by_commit_id(commit_id)?; + + // root..=new_root's parent + let to_commit = self.find_path(slab_index); + + // early return if new_root == root + if to_commit.is_empty() { + return Ok(( + self.nodes[slab_index].get_height() - to_commit.len(), + to_commit, + )); + } + + // subtree of new_root + let to_maintain_vec = self.bfs_subtree(slab_index); + let to_maintain = BTreeSet::from_iter(to_maintain_vec); + + // remove: tree - subtree of new_root + let mut to_remove = Vec::new(); + for (idx, _) in self.nodes.iter() { + if !to_maintain.contains(&idx) { + to_remove.push(idx); + } + } + for idx in to_remove { + self.index_map + .remove(&self.nodes.remove(idx).get_commit_id()); + } + + // set new_root's parent as None + self.nodes[slab_index].set_parent(None); + + // (height of new_root's parent, root..=new_root's parent) + Ok(( + self.nodes[slab_index].get_height() - to_commit.len(), + to_commit, + )) + } +} diff --git a/src/middlewares/versioned_flat_key_value/pending_part/tree/checkout_support.rs b/src/middlewares/versioned_flat_key_value/pending_part/tree/checkout_support.rs new file mode 100644 index 0000000..7448b40 --- /dev/null +++ b/src/middlewares/versioned_flat_key_value/pending_part/tree/checkout_support.rs @@ -0,0 +1,83 @@ +use std::collections::BTreeMap; + +use crate::middlewares::versioned_flat_key_value::pending_part::pending_schema::{ + ApplyMap, ApplyRecord, PendingKeyValueSchema, Result as PendResult, +}; + +use super::Tree; + +// methods to support VersionedMap::checkout_current() +impl Tree { + pub fn get_apply_map_from_root_included( + &self, + target_commit_id: S::CommitId, + ) -> PendResult, S> { + let mut target_node = self.get_node_by_commit_id(target_commit_id)?; + let mut commits_rev = BTreeMap::new(); + target_node.export_commit_data(&mut commits_rev); + while let Some(parent_slab_index) = target_node.get_parent() { + target_node = self.get_node_by_slab_index(parent_slab_index); + target_node.export_commit_data(&mut commits_rev); + } + Ok(commits_rev) + } + + // correctness based on single root + #[allow(clippy::type_complexity)] + pub fn collect_rollback_and_apply_ops( + &self, + current_commit_id: S::CommitId, + target_commit_id: S::CommitId, + ) -> PendResult<(BTreeMap>>, ApplyMap), S> { + let mut current_node = self.get_node_by_commit_id(current_commit_id).unwrap(); + let mut target_node = self.get_node_by_commit_id(target_commit_id)?; + let mut rollbacks = BTreeMap::new(); + let mut commits_rev = BTreeMap::new(); + while current_node.get_height() > target_node.get_height() { + current_node.export_rollback_data(&mut rollbacks); + current_node = self.get_parent_node(current_node).unwrap(); + } + while target_node.get_height() > current_node.get_height() { + target_node.export_commit_data(&mut commits_rev); + target_node = self.get_parent_node(target_node).unwrap(); + } + while current_node.get_commit_id() != target_node.get_commit_id() { + current_node.export_rollback_data(&mut rollbacks); + current_node = self.get_parent_node(current_node).unwrap(); + target_node.export_commit_data(&mut commits_rev); + target_node = self.get_parent_node(target_node).unwrap(); + } + // check rollbacks' old_commit_id because TreeNodes are deleted + // in a lazy way with respect to TreeNodes.modifications + // todo: test this lazy method + for (_, old_commit_id_option) in rollbacks.iter_mut() { + if let Some(ref old_commit_id) = old_commit_id_option { + if !self.contains_commit_id(old_commit_id) { + *old_commit_id_option = None; + } + } + } + + let rollbacks_with_value: BTreeMap<_, _> = rollbacks + .into_iter() + .map(|(k, old_cid_opt)| match old_cid_opt { + None => (k, None), + Some(rollback_cid) => { + let rollback_value = self.get_by_commit_id(rollback_cid, &k).unwrap().unwrap(); + ( + k, + Some(ApplyRecord:: { + value: rollback_value, + commit_id: rollback_cid, + }), + ) + } + }) + .collect(); + + // rollbacks or commits_rev may be empty, + // they contain current and target (if they are not lca), respectively, + // but they do not contain lca + Ok((rollbacks_with_value, commits_rev)) + } +} diff --git a/src/middlewares/versioned_flat_key_value/pending_part/tree/key_value_store_manager_support.rs b/src/middlewares/versioned_flat_key_value/pending_part/tree/key_value_store_manager_support.rs new file mode 100644 index 0000000..6162de6 --- /dev/null +++ b/src/middlewares/versioned_flat_key_value/pending_part/tree/key_value_store_manager_support.rs @@ -0,0 +1,61 @@ +use crate::middlewares::{ + versioned_flat_key_value::pending_part::pending_schema::{ + PendingKeyValueSchema, Result as PendResult, + }, + PendingError, +}; + +use super::Tree; + +// Internal Tree methods supporting +// helper methods in VersionedMap for +// implementing `KeyValueStoreManager` for `VersionedStore`. +impl Tree { + pub fn iter_historical_changes<'a>( + &'a self, + commit_id: &S::CommitId, + key: &'a S::Key, + ) -> PendResult)>, S> { + let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); + let mut path = Vec::new(); + while let Some(node) = node_option { + if let Some(value) = node.get_modified_value(key) { + path.push((node.get_commit_id(), key, value)); + } + node_option = self.get_parent_node(node); + } + Ok(path.into_iter()) + } + + pub fn get_versioned_key( + &self, + commit_id: &S::CommitId, + key: &S::Key, + ) -> PendResult>, S> { + let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); + while let Some(node) = node_option { + if let Some(value) = node.get_modified_value(key) { + return Ok(Some(value)); + } + node_option = self.get_parent_node(node); + } + Ok(None) + } + + pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult<(), S> { + let slab_index = self.get_slab_index_by_commit_id(commit_id)?; + let node = self.get_node_by_slab_index(slab_index); + if let Some(parent_of_discard) = node.get_parent() { + let to_remove = self.bfs_subtree(slab_index); + for idx in to_remove { + self.index_map + .remove(&self.nodes.remove(idx).get_commit_id()); + } + let parent_node = self.get_mut_node_by_slab_index(parent_of_discard); + parent_node.remove_child(&slab_index); + Ok(()) + } else { + Err(PendingError::RootShouldNotBeDiscarded) + } + } +} diff --git a/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs b/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs index a2ad8ca..99625e4 100644 --- a/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs +++ b/src/middlewares/versioned_flat_key_value/pending_part/tree/mod.rs @@ -1,16 +1,18 @@ +mod add_node_support; +mod change_root_support; +mod checkout_support; +mod key_value_store_manager_support; mod node; pub type SlabIndex = usize; -use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; +use std::collections::HashMap; use slab::Slab; use self::node::TreeNode; -use super::pending_schema::{ - ApplyMap, ApplyRecord, KeyValueMap, PendingKeyValueSchema, RecoverMap, Result as PendResult, -}; +use super::pending_schema::{PendingKeyValueSchema, Result as PendResult}; use super::PendingError; pub struct Tree { @@ -89,245 +91,3 @@ impl Tree { slab_indices } } - -// methods to support VersionedMap::add_node() -impl Tree { - pub fn add_root( - &mut self, - commit_id: S::CommitId, - modifications: RecoverMap, - ) -> PendResult<(), S> { - // return error if there is root - if self.has_root() { - return Err(PendingError::MultipleRootsNotAllowed); - } - // PendingError::CommitIdAlreadyExists(_) cannot happend because no root indicates no node - - // new root - let root = TreeNode::new_root(commit_id, modifications, self.height_of_root); - - // add root to tree - let slab_index = self.nodes.insert(root); - self.index_map.insert(commit_id, slab_index); - - Ok(()) - } - - pub fn add_non_root_node( - &mut self, - commit_id: S::CommitId, - parent_commit_id: S::CommitId, - modifications: RecoverMap, - ) -> PendResult<(), S> { - // return error if parent_commit_id does not exist - let parent_slab_index = self.get_slab_index_by_commit_id(parent_commit_id)?; - - // return error if commit_id exists - if self.contains_commit_id(&commit_id) { - return Err(PendingError::CommitIdAlreadyExists(commit_id)); - } - - // new node - let node = TreeNode::new_non_root( - commit_id, - parent_slab_index, - self.get_node_by_slab_index(parent_slab_index).get_height() + 1, - modifications, - ); - - // add node to tree - let slab_index = self.nodes.insert(node); - self.index_map.insert(commit_id, slab_index); - self.nodes[parent_slab_index].insert_child(slab_index); - - Ok(()) - } -} - -// methods to support VersionedMap::change_root() -impl Tree { - // excluding target - fn find_path(&self, target_slab_index: SlabIndex) -> Vec<(S::CommitId, KeyValueMap)> { - let mut target_node = self.get_node_by_slab_index(target_slab_index); - let mut path = VecDeque::new(); - while let Some(parent_slab_index) = target_node.get_parent() { - target_node = self.get_node_by_slab_index(parent_slab_index); - path.push_front((target_node.get_commit_id(), target_node.get_updates())); - } - path.into() - } - - // todo: test - #[allow(clippy::type_complexity)] - pub fn change_root( - &mut self, - commit_id: S::CommitId, - ) -> PendResult<(usize, Vec<(S::CommitId, KeyValueMap)>), S> { - let slab_index = self.get_slab_index_by_commit_id(commit_id)?; - - // root..=new_root's parent - let to_commit = self.find_path(slab_index); - - // early return if new_root == root - if to_commit.is_empty() { - return Ok(( - self.nodes[slab_index].get_height() - to_commit.len(), - to_commit, - )); - } - - // subtree of new_root - let to_maintain_vec = self.bfs_subtree(slab_index); - let to_maintain = BTreeSet::from_iter(to_maintain_vec); - - // remove: tree - subtree of new_root - let mut to_remove = Vec::new(); - for (idx, _) in self.nodes.iter() { - if !to_maintain.contains(&idx) { - to_remove.push(idx); - } - } - for idx in to_remove { - self.index_map - .remove(&self.nodes.remove(idx).get_commit_id()); - } - - // set new_root's parent as None - self.nodes[slab_index].set_parent(None); - - // (height of new_root's parent, root..=new_root's parent) - Ok(( - self.nodes[slab_index].get_height() - to_commit.len(), - to_commit, - )) - } -} - -// Internal Tree methods supporting -// helper methods in VersionedMap for -// implementing `KeyValueStoreManager` for `VersionedStore`. -impl Tree { - pub fn iter_historical_changes<'a>( - &'a self, - commit_id: &S::CommitId, - key: &'a S::Key, - ) -> PendResult)>, S> { - let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); - let mut path = Vec::new(); - while let Some(node) = node_option { - if let Some(value) = node.get_modified_value(key) { - path.push((node.get_commit_id(), key, value)); - } - node_option = self.get_parent_node(node); - } - Ok(path.into_iter()) - } - - pub fn get_versioned_key( - &self, - commit_id: &S::CommitId, - key: &S::Key, - ) -> PendResult>, S> { - let mut node_option = Some(self.get_node_by_commit_id(*commit_id)?); - while let Some(node) = node_option { - if let Some(value) = node.get_modified_value(key) { - return Ok(Some(value)); - } - node_option = self.get_parent_node(node); - } - Ok(None) - } - - pub fn discard(&mut self, commit_id: S::CommitId) -> PendResult<(), S> { - let slab_index = self.get_slab_index_by_commit_id(commit_id)?; - let node = self.get_node_by_slab_index(slab_index); - if let Some(parent_of_discard) = node.get_parent() { - let to_remove = self.bfs_subtree(slab_index); - for idx in to_remove { - self.index_map - .remove(&self.nodes.remove(idx).get_commit_id()); - } - let parent_node = self.get_mut_node_by_slab_index(parent_of_discard); - parent_node.remove_child(&slab_index); - Ok(()) - } else { - Err(PendingError::RootShouldNotBeDiscarded) - } - } -} - -// methods to support VersionedMap::checkout_current() -impl Tree { - pub fn get_apply_map_from_root_included( - &self, - target_commit_id: S::CommitId, - ) -> PendResult, S> { - let mut target_node = self.get_node_by_commit_id(target_commit_id)?; - let mut commits_rev = BTreeMap::new(); - target_node.export_commit_data(&mut commits_rev); - while let Some(parent_slab_index) = target_node.get_parent() { - target_node = self.get_node_by_slab_index(parent_slab_index); - target_node.export_commit_data(&mut commits_rev); - } - Ok(commits_rev) - } - - // correctness based on single root - #[allow(clippy::type_complexity)] - pub(super) fn collect_rollback_and_apply_ops( - &self, - current_commit_id: S::CommitId, - target_commit_id: S::CommitId, - ) -> PendResult<(BTreeMap>>, ApplyMap), S> { - let mut current_node = self.get_node_by_commit_id(current_commit_id).unwrap(); - let mut target_node = self.get_node_by_commit_id(target_commit_id)?; - let mut rollbacks = BTreeMap::new(); - let mut commits_rev = BTreeMap::new(); - while current_node.get_height() > target_node.get_height() { - current_node.export_rollback_data(&mut rollbacks); - current_node = self.get_parent_node(current_node).unwrap(); - } - while target_node.get_height() > current_node.get_height() { - target_node.export_commit_data(&mut commits_rev); - target_node = self.get_parent_node(target_node).unwrap(); - } - while current_node.get_commit_id() != target_node.get_commit_id() { - current_node.export_rollback_data(&mut rollbacks); - current_node = self.get_parent_node(current_node).unwrap(); - target_node.export_commit_data(&mut commits_rev); - target_node = self.get_parent_node(target_node).unwrap(); - } - // check rollbacks' old_commit_id because TreeNodes are deleted - // in a lazy way with respect to TreeNodes.modifications - // todo: test this lazy method - for (_, old_commit_id_option) in rollbacks.iter_mut() { - if let Some(ref old_commit_id) = old_commit_id_option { - if !self.contains_commit_id(old_commit_id) { - *old_commit_id_option = None; - } - } - } - - let rollbacks_with_value: BTreeMap<_, _> = rollbacks - .into_iter() - .map(|(k, old_cid_opt)| match old_cid_opt { - None => (k, None), - Some(rollback_cid) => { - let rollback_value = self.get_by_commit_id(rollback_cid, &k).unwrap().unwrap(); - ( - k, - Some(ApplyRecord:: { - value: rollback_value, - commit_id: rollback_cid, - }), - ) - } - }) - .collect(); - - // rollbacks or commits_rev may be empty, - // they contain current and target (if they are not lca), respectively, - // but they do not contain lca - Ok((rollbacks_with_value, commits_rev)) - } -}