From 088dc0093243cf51eb388a4b131de2f3854c2e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 21 Jul 2023 17:06:35 +0200 Subject: [PATCH 1/9] fix: make compact callbacks async - `update_lines` - `list_removed_locations` - `filter_removed_locations` Fix doc mistake for `list_removed_locations` (negation was missing) --- src/callbacks.rs | 8 ++++---- src/compact.rs | 19 +++++++++++-------- src/compact_live.rs | 2 +- src/in_memory_example.rs | 6 +++--- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/callbacks.rs b/src/callbacks.rs index 426a259f..3d7c53e7 100644 --- a/src/callbacks.rs +++ b/src/callbacks.rs @@ -143,14 +143,14 @@ pub trait FindexCallbacks, new_entry_table_items: EncryptedTable, new_chain_table_items: EncryptedTable, ) -> Result<(), Error>; - /// Returns all locations that point to existing data. + /// Returns all locations that do not point to existing data. /// /// **NOTE**: this callback does not call the index database since indexed /// locations can be anything in Findex (DB UID, path, other ID...). It may @@ -159,14 +159,14 @@ pub trait FindexCallbacks, ) -> Result, Error>; #[cfg(feature = "live_compact")] /// Returns all locations that point to existing data. - fn filter_removed_locations( + async fn filter_removed_locations( &self, locations: HashSet, ) -> Result, Error>; diff --git a/src/compact.rs b/src/compact.rs index 0c45d0d0..3d818b79 100644 --- a/src/compact.rs +++ b/src/compact.rs @@ -155,13 +155,15 @@ pub trait FindexCompact< // calling for one location batch for one word to add noise and prevent // the database the size of the chains for each keywords. // - let removed_locations = self.list_removed_locations( - reindexed_chain_values - .values() - .flat_map(|chain| chain.iter().filter_map(IndexedValue::get_location)) - .cloned() - .collect(), - )?; + let removed_locations = self + .list_removed_locations( + reindexed_chain_values + .values() + .flat_map(|chain| chain.iter().filter_map(IndexedValue::get_location)) + .cloned() + .collect(), + ) + .await?; for entry_table_uid in entry_table_items_to_reindex { let entry_table_value = entry_table.get(&entry_table_uid).ok_or_else(|| { @@ -243,7 +245,8 @@ pub trait FindexCompact< chain_table_uids_to_remove, entry_table.encrypt(&mut rng, &new_k_value)?, chain_table_adds, - )?; + ) + .await?; Ok(()) } diff --git a/src/compact_live.rs b/src/compact_live.rs index 5f694751..6d201a72 100644 --- a/src/compact_live.rs +++ b/src/compact_live.rs @@ -355,7 +355,7 @@ pub trait FindexLiveCompact< .filter_map(IndexedValue::get_location) .cloned() .collect(); - let noisy_remaining_locations = self.filter_removed_locations(noisy_locations)?; + let noisy_remaining_locations = self.filter_removed_locations(noisy_locations).await?; // All fetched Entry Table entries need to be modified. while !encrypted_entry_table.is_empty() { diff --git a/src/in_memory_example.rs b/src/in_memory_example.rs index 8ae33f80..d9178499 100644 --- a/src/in_memory_example.rs +++ b/src/in_memory_example.rs @@ -198,7 +198,7 @@ impl FindexCallbacks Ok(()) } - fn update_lines( + async fn update_lines( &mut self, chain_table_uids_to_remove: Uids, new_encrypted_entry_table_items: EncryptedTable, @@ -220,7 +220,7 @@ impl FindexCallbacks Ok(()) } - fn list_removed_locations( + async fn list_removed_locations( &self, _: HashSet, ) -> Result, ExampleError> { @@ -228,7 +228,7 @@ impl FindexCallbacks } #[cfg(feature = "live_compact")] - fn filter_removed_locations( + async fn filter_removed_locations( &self, locations: HashSet, ) -> Result, ExampleError> { From e605ed0d056e3279bc3944808aa4fc0768823329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 21 Jul 2023 17:10:40 +0200 Subject: [PATCH 2/9] fix: upgrade version and fill changelog --- CHANGELOG.md | 7 +++++++ Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe28824..1cd7e09b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. +## [5.0.0] - 2023-07-21 + +### Bug Fixes + +- add missing `async` keyword for compact callbacks +- fix `list_removed_locations` doc + ## [4.0.3] - 2023-07-11 ### Features diff --git a/Cargo.toml b/Cargo.toml index fd2c7bd0..bca8a4ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cosmian_findex" -version = "4.0.3" +version = "5.0.0" authors = [ "Chloé Hébant ", "Bruno Grieder ", From 81039cb74f8b895d4612abcfde587eb428382ee9 Mon Sep 17 00:00:00 2001 From: Manuthor Date: Sat, 22 Jul 2023 09:38:38 +0200 Subject: [PATCH 3/9] fix: cargo license file --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index bca8a4ba..36f99259 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ authors = [ categories = ["cosmian::crypto"] edition = "2021" keywords = ["SSE"] -license = "MIT/Apache-2.0" +license-file = "LICENSE.md" repository = "https://github.com/Cosmian/findex/" description = "Symmetric Searchable Encryption" From d3b6673c88aad8840156b68dce8341e011402f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Thu, 27 Jul 2023 17:02:16 +0200 Subject: [PATCH 4/9] feat: remove live compact --- Cargo.toml | 1 - src/compact_live.rs | 446 --------------------------------------- src/in_memory_example.rs | 46 +--- src/lib.rs | 5 - 4 files changed, 5 insertions(+), 493 deletions(-) delete mode 100644 src/compact_live.rs diff --git a/Cargo.toml b/Cargo.toml index 36f99259..89a700e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ path = "src/lib.rs" [features] in_memory = [] -live_compact = [] [dependencies] base64 = "0.21.2" diff --git a/src/compact_live.rs b/src/compact_live.rs deleted file mode 100644 index 6d201a72..00000000 --- a/src/compact_live.rs +++ /dev/null @@ -1,446 +0,0 @@ -//! Defines the live compact trait. - -use std::collections::{HashMap, HashSet}; - -use cosmian_crypto_core::{ - reexport::rand_core::{CryptoRngCore, SeedableRng}, - CsRng, -}; - -use crate::{ - chain_table::KwiChainUids, - entry_table::{EntryTable, EntryTableValue}, - parameters::DemKey, - structs::{BlockType, ChainData}, - CallbackError, EncryptedTable, Error, FindexCallbacks, FindexCompact, IndexedValue, - KeyingMaterial, Location, Uid, Uids, UpsertData, CHAIN_TABLE_KEY_DERIVATION_INFO, - ENTRY_TABLE_KEY_DERIVATION_INFO, -}; - -/// [Euler's constant](https://wikipedia.org/wiki/Euler_constant). -const GAMMA: f64 = 0.577; - -/// Live compact trait allows for compacting chains associated to a set of Entry -/// Table UIDs without interrupting the service. -/// -/// The compaction of a chain removes duplicated entries, simplifies deletions -/// (removes all addition of the deleted keyword up to the deletion and removes -/// the deletion) and write remaining values in a compact way in the Chain Table -/// (only the last line of the chain may have padding blocks). -/// -/// The following gives an example of the effect of a compact operation on an -/// Entry Table line and its associated chain. -/// -/// The Entry Table line: -/// -/// | UID | -> | H(w) | `K_w` | `UID_5` | -/// -/// becomes: -/// -/// | UID | -> | H(w) | `K_w`' | `UID_1`' | -/// -/// and the old chain (values are chosen arbitrarily in the aim to illustrate -/// different properties and `B1` ... `B4` are the non-padding blocks indexed -/// for the keyword `w`): -/// -/// | `UID_1` | -> Enc(`K_w`, | B1 (Add) | B2 (Add) | Padding |) -/// | `UID_2` | -> Enc(`K_w`, | B2 (Add) | Padding | Padding |) -/// | `UID_3` | -> Enc(`K_w`, | B1 (Del) | B3 (Add) | B4 (Add) |) -/// | `UID_4` | -> Enc(`K_w`, | B2 (Del) | Padding | Padding |) -/// | `UID_5` | -> Enc(`K_w`, | B2 (Add) | B3 (Add) | B4 (Add) |) -/// -/// becomes: -/// -/// | `UID_1`' | -> Enc(`K_w`', | B2 (Add) | B3 (Add) | B4 (Add) |) -/// -/// The following operations have been applied: -/// - a new random ephemeral key `K_w'` is drawn; -/// - the new last chain UID is stored in the Entry Table entry; -/// - chain UIDs are randomly generated; -/// - chain values are encrypted under the new ephemeral key; -/// - `B1` is removed since its last occurrence in the chain is a deletion; -/// - `B2` is kept since its last occurrence is an addition and is simplified -/// (only one occurrence in the chain); -/// - `B3` and `B4` are kept since they have not been deleted, and simplified. -/// -/// *NOTE*: the order of the locations in the compacted chain is not guaranteed. -pub trait FindexLiveCompact< - const UID_LENGTH: usize, - const BLOCK_LENGTH: usize, - const CHAIN_TABLE_WIDTH: usize, - const MASTER_KEY_LENGTH: usize, - const KWI_LENGTH: usize, - const KMAC_KEY_LENGTH: usize, - CustomError: std::error::Error + CallbackError, ->: - FindexCallbacks - // to get `batch_fetch_chains()` - + FindexCompact< - UID_LENGTH, - BLOCK_LENGTH, - CHAIN_TABLE_WIDTH, - MASTER_KEY_LENGTH, - KWI_LENGTH, - KMAC_KEY_LENGTH, - CustomError, - > -{ - /// Ratio of the noise over the real data. - const NOISE_RATIO: f64; - - /// Number of mixed Entry Table UIDs (target UIDs or noise UIDs) to compact in a row. - const BATCH_SIZE: usize; - - /// Selects a set of Entry Table UIDs to reindex (target set) and a set of Entry Table UIDs - /// used as noise (noise set). - /// - /// These sets are mutually exclusive and the ratio of the cardinal of the noise set over the - /// target set is given by `Self::NOISE_RATIO`. - /// - /// Returns the target set and the noise set. A `HashSet` is used for the noise set to allow fast - /// lookups and a list is used for the target set to avoid hashing. - /// - /// # Parameters - /// - /// - `rng` : secure random number generator - /// - `entry_table_uids` : Entry Table uids - /// - `num_reindexing_before_full_set` : number of compact operations need to compact all the - /// base - async fn select_uids_with_noise( - &self, - rng: &mut impl CryptoRngCore, - num_reindexing_before_full_set: u32 - ) -> Result<(Vec>, Uids), Error> { - - let entry_table_uids = self.fetch_all_entry_table_uids().await?; - - - if Self::NOISE_RATIO > 1f64 { - return Err(Error::CryptoError(format!( - "noise ratio cannot be greater than 1 ({} given)", - Self::NOISE_RATIO - ))); - } - - // The esperance of the number of UIDs to draw in order to recompact the - // full Entry Table after `num_reindexing_before_full_set` operations: - // - // `n_compact = (n / T) * (log(n) + gamma) + O(1/(T * n))` - // - // with `n = entry_table_length`, `T = num_reindexing_before_full_set` - // and `gamma` is the Euler's constant. - // - // See the [coupon collector's problem](https://wikipedia.org/wiki/Coupon_collector's_problem). - let entry_table_length = entry_table_uids.len() as f64; - let n_compact = ((entry_table_length * (entry_table_length.log2() + GAMMA)) - / f64::from(num_reindexing_before_full_set)) - .ceil() as usize; - - // The number of compacted UIDs should leave enough unused UIDs for the noise. - if (n_compact as f64 * (1f64 + Self::NOISE_RATIO)) > entry_table_uids.len() as f64 { - return Err(Error::CryptoError(format!( - "Number of Entry Table UIDs to compact ({n_compact}) should not be greater than {}", - entry_table_uids.len() as f64 / (1f64 + Self::NOISE_RATIO) - ))); - } - - // Upper bounds are used for capacity in order to avoid reallocations: - // - the number of noise UIDs is lower than `n_compact`; - let mut noise_uids = HashSet::with_capacity(n_compact); - // - the total number of UIDs (target set + noise set) is lower than `2*n_compact`. - let mut mixed_uids = Vec::with_capacity(2 * n_compact); - - // Needed because `uids` is moved in the loop condition. - let n_uids = entry_table_uids.len(); - let n_noise_candidates = n_uids - n_compact; - let n_noise = (Self::NOISE_RATIO * n_compact as f64) as usize; - - for uid in entry_table_uids { - let tmp = rng.next_u32() as usize; - if tmp % n_uids < n_compact { - // Randomly select ~ `n_compact` target UIDs. - mixed_uids.push(uid); - } else if tmp % n_noise_candidates < n_noise { - // Randomly select ~ `NOISE_RATIO * n_compact` noise UIDs. - noise_uids.insert(uid); - mixed_uids.push(uid); - } - } - - Ok((mixed_uids, Uids(noise_uids))) - } - - /// Fetch all useful information for the compact from the Chain Table: - /// - the map of Entry Table UIDs to Chain Table UIDs - /// - the map of Entry Table UIDs to indexed values - /// - /// # Parameters - /// - /// - `k_value` : DEM key used to decrypt the Entry Table - /// - `encrypted_entry_table` : encrypted Entry Table - async fn fetch_chain_data( - &self, - k_value: &DemKey, - encrypted_entry_table: &EncryptedTable, - ) -> Result< - ChainData, - Error, - > { - let entry_table: EntryTable = - EntryTable::decrypt(k_value, encrypted_entry_table)?; - - let mut chains = ChainData::with_capacity(entry_table.len()); - - // Unchain all Entry Table UIDs. - let kwi_chain_table_uids: KwiChainUids = entry_table.iter().map(|(_, value)| -> Result<_, _>{ - let k_uid = value.kwi.derive_kmac_key::(CHAIN_TABLE_KEY_DERIVATION_INFO); - Ok(( - value.kwi.clone(), - value.unchain::< - CHAIN_TABLE_WIDTH, - BLOCK_LENGTH, - KMAC_KEY_LENGTH, - >(&k_uid)? - )) - }).collect::>>()?; - - // Associate Entry Table UIDs to Chain Table UIDs. - for (uid, v) in entry_table.iter() { - chains.chain_uids.insert( - *uid, - kwi_chain_table_uids - .get(&v.kwi) - .ok_or(Error::::CryptoError(format!( - "no matching Kwi in `kwi_chain_table_uids` ({:?})", - v.kwi - )))?.clone(), - ); - } - - // Fetch the Chain Table values for the chain UIDs. - let chain_values = self - .fetch_chains(kwi_chain_table_uids) - .await?; - - // Convert the blocks of the given chains into indexed values. - for (entry_table_uid, entry_table_value) in entry_table { - let chain = - chain_values - .get(&entry_table_value.kwi) - .ok_or(Error::::CryptoError(format!( - "no chain found for the `Kwi`: {:?}", - entry_table_value.kwi - )))?; - let blocks = chain - .iter() - .flat_map(|(_, chain_table_value)| chain_table_value.as_blocks()); - chains.chain_values - .insert(entry_table_uid, IndexedValue::from_blocks(blocks)?); - } - - Ok(chains) - } - - /// Updates noisy Entry Table entries and compacts chains associated to - /// targeted entries. - /// - /// Returns the new Entry Table entries with noise entries unchanged, but - /// updated `K_wi` and last UID value for the targeted ones, along with - /// their associated encrypted and compacted Chain Table entries. - /// - /// # Parameters - /// - /// - `rng` : secure random number generator - /// - `k_value` : DEM key used to decrypt the Entry Table - /// - `noise` : Entry Table UIDs used as noise - /// - `noisy_remaining_locations` : remaining locations (contains noise) - /// - `noisy_encrypted_entry_table` : encrypted Entry Table (contains noise) - /// - `noisy_chain_values` : chain values (contains noise) - fn compact_chains( - &self, - rng: &mut impl CryptoRngCore, - k_value: &DemKey, - noise: &HashSet>, - noisy_remaining_locations: &HashSet, - noisy_encrypted_entry_table: &EncryptedTable, - noisy_chain_values: &HashMap, HashSet>, - ) -> Result< - ( - EntryTable, - HashMap, EncryptedTable>, - ), - Error, - > { - let mut noisy_entry_table = EntryTable::with_capacity(noisy_chain_values.len()); - let mut compacted_chains = HashMap::with_capacity(noisy_chain_values.len() - noise.len()); - let mut cache = HashMap::with_capacity(noisy_chain_values.len() - noise.len()); - - for (uid, encrypted_value) in noisy_encrypted_entry_table.iter() { - let entry_table_value = EntryTableValue::decrypt( - k_value, - encrypted_value - )?; - if noise.contains(uid) { - // Noise entries are simply re-encrypted. - noisy_entry_table.insert(*uid, entry_table_value); - } else { - // Compact chains associated to the other entries. - let chain = noisy_chain_values.get(uid).ok_or(Error::::CryptoError("No matching UID in chains.".to_string()))?; - let mut compacted_chain = HashMap::with_capacity(chain.len()); - for value in chain { - match value { - IndexedValue::Location(location) => { - if noisy_remaining_locations.contains(location) { - // Add remaining locations back. All deletions should have been - // filtered by the fetch. - compacted_chain.insert( - IndexedValue::from(location.clone()), - BlockType::Addition, - ); - } - } - IndexedValue::NextKeyword(keyword) => { - // TODO (TBZ): deal with obsolete `NextWord`s. For now just add the - // `NextWord` again. - compacted_chain - .insert(IndexedValue::from(keyword.clone()), BlockType::Addition); - } - }; - } - compacted_chains.insert(*uid, (entry_table_value.keyword_hash, compacted_chain)); - cache.insert(entry_table_value.keyword_hash, *uid); - } - } - - let compacted_chain_table = noisy_entry_table.upsert::< - CHAIN_TABLE_WIDTH, - BLOCK_LENGTH, - KMAC_KEY_LENGTH, - >(rng, &compacted_chains)?; - - Ok((noisy_entry_table, compacted_chain_table)) - } - - /// Live compact the given UIDs. - /// - /// - `rng` : secure random number generator - /// - `k_value` : DEM key used to decrypt the Entry Table - /// - `noisy_uids` : UIDs to compact (contains noise) - /// - `noise` : Entry Table UIDs used as noise - async fn live_compact_uids( - &mut self, - rng: &mut impl CryptoRngCore, - k_value: &DemKey, - mixed_uids: HashSet>, - noise_uids: &HashSet>, - ) -> Result<(), Error> { - // Fetch both target and noise values from the Entry Table. - let mut encrypted_entry_table = self - .fetch_entry_table(Uids(mixed_uids)) - .await? - .try_into()?; - - // Fetch all associated chains. - let mut chains = self - .fetch_chain_data( - k_value, - &encrypted_entry_table, - ) - .await?; - - // Select remaining locations in both target and noise chains. - let noisy_locations = chains.chain_values - .iter() - .flat_map(|(_, values)| values) - .filter_map(IndexedValue::get_location) - .cloned() - .collect(); - let noisy_remaining_locations = self.filter_removed_locations(noisy_locations).await?; - - // All fetched Entry Table entries need to be modified. - while !encrypted_entry_table.is_empty() { - // Compute the new Entry Table (target + noise) and compacted chains (target only). - let (noisy_entry_table, new_chains) = self.compact_chains( - rng, - k_value, - noise_uids, - &noisy_remaining_locations, - &encrypted_entry_table, - &chains.chain_values, - )?; - - // Insert all recompacted chains. - self.insert_chain_table(new_chains.iter().flat_map(|(_, v)| v.clone()).collect()).await?; - - // Try upserting the new Entry Table. - let upsert_data = UpsertData::new( - &encrypted_entry_table, - noisy_entry_table.encrypt(rng, k_value)?, - ); - - // Delete unused chains (at least one chain value per entry line): - let mut chains_to_delete = HashSet::with_capacity(upsert_data.len()); - - // These are failures to upsert. - encrypted_entry_table = self.upsert_entry_table(upsert_data).await?; - - // - new chains corresponding to unsuccessful upserts - for (uid, chain) in new_chains { - if encrypted_entry_table.contains_key(&uid) { - chains_to_delete.extend(chain.into_keys()); - } - } - - // - old chains corresponding to successful upserts - // (the following clones the Entry Table UIDs rather than cloning the chain UIDs - // which leads to the same number of clone operations, or less) - let mut old_chains_to_remove = Vec::with_capacity(chains.chain_uids.len()); - for uid in chains.chain_uids.keys() { - if !encrypted_entry_table.contains_key(uid) && !noise_uids.contains(uid) { - old_chains_to_remove.push(*uid); - } - } - for uid in old_chains_to_remove { - let chain = chains.chain_uids.remove(&uid).expect( - "This will never happens since selected UIDs come from `chains.chain_uids`." - ); - chains_to_delete.extend(chain); - } - - self.delete_chain(Uids(chains_to_delete)).await?; - } - - Ok(()) - } - - /// Compacts a random subset of the Entry Table. - /// - /// The size of this subset is such that `num_reindexing_before_full_set` compact operations - /// are needed to compact all the Entry Table. - /// - /// - `master_key` : Findex master key - /// - `num_reindexing_before_full_set` : number of compact operations needed to compact all - /// the Entry Table - async fn live_compact( - &mut self, - master_key: &KeyingMaterial, - num_reindexing_before_full_set: u32, - ) -> Result<(), Error> { - let mut rng = CsRng::from_entropy(); - let k_value = master_key.derive_dem_key(ENTRY_TABLE_KEY_DERIVATION_INFO); - - let (mixed_uids, noise_uids) = - self.select_uids_with_noise(&mut rng, num_reindexing_before_full_set).await?; - - for batch in mixed_uids.chunks(Self::BATCH_SIZE) { - self.live_compact_uids( - &mut rng, - &k_value, - batch.iter().copied().collect(), - &noise_uids, - ).await?; - } - - Ok(()) - } -} diff --git a/src/in_memory_example.rs b/src/in_memory_example.rs index d9178499..04216cd0 100644 --- a/src/in_memory_example.rs +++ b/src/in_memory_example.rs @@ -11,14 +11,11 @@ use cosmian_crypto_core::{ use rand::Rng; use crate::{ - callbacks::FetchChains, parameters::UID_LENGTH, structs::EncryptedMultiTable, EncryptedTable, - FindexCallbacks, FindexCompact, FindexSearch, FindexUpsert, IndexedValue, Keyword, Location, - Uids, UpsertData, -}; -#[cfg(feature = "live_compact")] -use crate::{ - compact_live::FindexLiveCompact, - parameters::{BLOCK_LENGTH, CHAIN_TABLE_WIDTH, KMAC_KEY_LENGTH, KWI_LENGTH, MASTER_KEY_LENGTH}, + callbacks::FetchChains, + parameters::{BLOCK_LENGTH, CHAIN_TABLE_WIDTH, KWI_LENGTH, UID_LENGTH}, + structs::EncryptedMultiTable, + EncryptedTable, FindexCallbacks, FindexCompact, FindexSearch, FindexUpsert, IndexedValue, + Keyword, Location, Uids, UpsertData, }; #[derive(Debug)] @@ -226,23 +223,6 @@ impl FindexCallbacks ) -> Result, ExampleError> { Ok(self.removed_locations.iter().cloned().collect()) } - - #[cfg(feature = "live_compact")] - async fn filter_removed_locations( - &self, - locations: HashSet, - ) -> Result, ExampleError> { - Ok(locations - .into_iter() - .filter(|location| !self.removed_locations.contains(location)) - .collect()) - } - - #[cfg(feature = "live_compact")] - async fn delete_chain(&mut self, uids: Uids) -> Result<(), ExampleError> { - self.chain_table.retain(|uid, _| !uids.contains(uid)); - Ok(()) - } } impl FetchChains @@ -255,19 +235,3 @@ impl_findex_trait!(FindexSearch, FindexInMemory, ExampleError); impl_findex_trait!(FindexUpsert, FindexInMemory, ExampleError); impl_findex_trait!(FindexCompact, FindexInMemory, ExampleError); - -#[cfg(feature = "live_compact")] -impl - FindexLiveCompact< - UID_LENGTH, - BLOCK_LENGTH, - CHAIN_TABLE_WIDTH, - MASTER_KEY_LENGTH, - KWI_LENGTH, - KMAC_KEY_LENGTH, - ExampleError, - > for FindexInMemory -{ - const BATCH_SIZE: usize = 10; - const NOISE_RATIO: f64 = 0.5; -} diff --git a/src/lib.rs b/src/lib.rs index ae763979..35b88b80 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,16 +38,11 @@ mod upsert; pub mod parameters; -#[cfg(feature = "live_compact")] -mod compact_live; - #[cfg(feature = "in_memory")] pub mod in_memory_example; pub use callbacks::{FetchChains, FindexCallbacks}; pub use compact::FindexCompact; -#[cfg(feature = "live_compact")] -pub use compact_live::FindexLiveCompact; pub use error::{CallbackError, CoreError, Error}; pub use keys::KeyingMaterial; pub use search::FindexSearch; From edec34cfb297d67a62fc42fcf42db8d3d3bc8600 Mon Sep 17 00:00:00 2001 From: Bruno Grieder Date: Sun, 30 Jul 2023 11:56:27 +0200 Subject: [PATCH 5/9] added previous presence indication to upsert --- .DS_Store | Bin 0 -> 6148 bytes Cargo.toml | 21 +- french_dataset.json | 1002 +++++++++++++++++++++++++++++++++++++++ src/upsert.rs | 69 ++- tests/test_in_memory.rs | 90 ++++ 5 files changed, 1152 insertions(+), 30 deletions(-) create mode 100644 .DS_Store create mode 100644 french_dataset.json diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..76f1144ad0c66984be340b470032996987b402c2 GIT binary patch literal 6148 zcmeHK%}T>S5Z-O8-BN@e6nb3nS}@vDEM7vbFJMFuDm5WNgE3p0)E-J9XMG``#OHBl zcLSF8;7P>Jz~-BspWU4gvOkP5KAeTejI|hJ0vaMmWrLu3qpM1k=d*0+Pj9hz zrBo7>x*uG}(R|$6KUZn)$7wXx32_)f$lXnxhH5@kvoy?fu4f#gAsXXWd$Bm}4LY*l zb(S5u7@RmA+3WU~%Z4~OJUY7^KPNA#dNX8l;9bj}#S-2@S*z>So2H3MAHi4VmGOkc z05L!e5Chx8fIS05b9+jrN{Int;D-$0{ve?tx)uwAdh37>UZ2rlLqq``-x7$@qHD1* z2p$k_QUOgWw@(ai(!nlmoNKW#Xwn(iGs8G`=JN5v_3U7mI-GIWAbp7eVqlels_C}y z{J(%-X5%A&HHCV_05R~-7~r*$HyT1w_H11%56@Z&?ExAJ#ucc5fWC4GfDY~>ZS6FE bi8{o&77K$o3fgr#AYBABA@m^zeu056xwcIv literal 0 HcmV?d00001 diff --git a/Cargo.toml b/Cargo.toml index 89a700e7..5eab67ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,11 +2,11 @@ name = "cosmian_findex" version = "5.0.0" authors = [ - "Chloé Hébant ", - "Bruno Grieder ", - "Célia Corsin ", - "Emmanuel Coste ", - "Théophile Brézot ", + "Chloé Hébant ", + "Bruno Grieder ", + "Célia Corsin ", + "Emmanuel Coste ", + "Théophile Brézot ", ] categories = ["cosmian::crypto"] edition = "2021" @@ -25,14 +25,21 @@ in_memory = [] [dependencies] base64 = "0.21.2" -cosmian_crypto_core = { version = "9.0.0", features = ["aes", "sha3", "ser"], default-features = false } +cosmian_crypto_core = { version = "9.0.0", features = [ + "aes", + "sha3", + "ser", +], default-features = false } rand = "0.8" tiny-keccak = { version = "2.0.2", features = ["kmac", "sha3"] } zeroize = "1.5" [dev-dependencies] actix-rt = "2.8" -criterion = { version = "0.4", features = ["html_reports", "async_futures"], default_features = false } +criterion = { version = "0.4", features = [ + "html_reports", + "async_futures", +], default_features = false } futures = "0.3" serde_json = "1.0" diff --git a/french_dataset.json b/french_dataset.json new file mode 100644 index 00000000..ebf9479f --- /dev/null +++ b/french_dataset.json @@ -0,0 +1,1002 @@ +[ + { + "firstName": "Barthélemy", + "lastName": "MEUNIER", + "phone": "06 26 07 18 36", + "email": "tlaine87@dumont.fr", + "country": "France", + "region": "Grand Est", + "employeeNumber": "24513-1783", + "security": "confidential" + }, + { + "firstName": "Aaron", + "lastName": "LECLERCQ", + "phone": "08 72 64 64 25", + "email": "jlaurent9@gautier.eu", + "country": "France", + "region": "Pays de la Loire", + "employeeNumber": "92480-3186", + "security": "confidential" + }, + { + "firstName": "Annabelle", + "lastName": "REY", + "phone": "09 32 88 69 26", + "email": "zguyot@nguyen.fr", + "country": "France", + "region": "Centre-Val de Loire", + "employeeNumber": "91301", + "security": "confidential" + }, + { + "firstName": "Nine", + "lastName": "GARNIER", + "phone": "02 01 46 40 62", + "email": "gonthierlemaire@paris.com", + "country": "France", + "region": "Occitanie", + "employeeNumber": "12601", + "security": "confidential" + }, + { + "firstName": "Médéric", + "lastName": "GAUTIER", + "phone": "01 57 90 71 41", + "email": "anaisleroux@paul.eu", + "country": "France", + "region": "Bretagne", + "employeeNumber": "69533-3597", + "security": "confidential" + }, + { + "firstName": "Jacqueline", + "lastName": "MOULIN", + "phone": "06 80 74 10 53", + "email": "gleroux9@lefevre.net", + "country": "France", + "region": "Corse", + "employeeNumber": "22706", + "security": "confidential" + }, + { + "firstName": "Mathurin", + "lastName": "POIRIER", + "phone": "04 93 14 57 35", + "email": "grobin57@fernandez.com", + "country": "France", + "region": "Grand Est", + "employeeNumber": "76458", + "security": "confidential" + }, + { + "firstName": "Émilie", + "lastName": "BARRE", + "phone": "07 40 48 65 75", + "email": "cmeunier8@olivier.fr", + "country": "France", + "region": "Grand Est", + "employeeNumber": "72777-7930", + "security": "confidential" + }, + { + "firstName": "Noël", + "lastName": "MAILLARD", + "phone": "05 08 92 26 61", + "email": "edumont45@lefebvre.com", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "22914", + "security": "confidential" + }, + { + "firstName": "Bernadette", + "lastName": "DUVAL", + "phone": "09 86 41 24 01", + "email": "ylouis@cousin.org", + "country": "France", + "region": "Normandie", + "employeeNumber": "91526", + "security": "confidential" + }, + { + "firstName": "Aliénor", + "lastName": "FRANCOIS", + "phone": "01 58 39 36 80", + "email": "nlaurent@roche.com", + "country": "France", + "region": "Bretagne", + "employeeNumber": "08276", + "security": "confidential" + }, + { + "firstName": "Alcime", + "lastName": "MARTY", + "phone": "01 80 74 08 46", + "email": "bguillot1@schneider.info", + "country": "France", + "region": "Corse", + "employeeNumber": "51800", + "security": "confidential" + }, + { + "firstName": "Ismérie", + "lastName": "BERGER", + "phone": "09 51 90 51 33", + "email": "rbourgeois8@guillot.eu", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "44218-8208", + "security": "confidential" + }, + { + "firstName": "César", + "lastName": "BLANC", + "phone": "07 14 51 93 93", + "email": "kboyer2@lefevre.com", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "22436", + "security": "confidential" + }, + { + "firstName": "Julia", + "lastName": "RODRIGUEZ", + "phone": "07 89 77 50 12", + "email": "opierre0@colin.eu", + "country": "France", + "region": "Pays de la Loire", + "employeeNumber": "32837", + "security": "confidential" + }, + { + "firstName": "Angilran", + "lastName": "BLANCHARD", + "phone": "07 03 17 37 73", + "email": "rroger50@aubert.net", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "19262", + "security": "confidential" + }, + { + "firstName": "Clélie", + "lastName": "GUILLAUME", + "phone": "03 07 85 19 62", + "email": "kbrun@laurent.info", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "70688", + "security": "confidential" + }, + { + "firstName": "Clarisse", + "lastName": "LECOMTE", + "phone": "09 25 76 37 60", + "email": "ylambert9@sanchez.info", + "country": "France", + "region": "Grand Est", + "employeeNumber": "99208", + "security": "confidential" + }, + { + "firstName": "Henriette", + "lastName": "LEROUX", + "phone": "00 39 00 63 65", + "email": "qjacquet66@dumas.info", + "country": "France", + "region": "Auvergne-Rhône-Alpes", + "employeeNumber": "72500-2982", + "security": "confidential" + }, + { + "firstName": "Laurine", + "lastName": "LECLERCQ", + "phone": "07 46 90 69 09", + "email": "wleclerc@dubois.org", + "country": "France", + "region": "Bourgogne-Franche-Comté", + "employeeNumber": "43573", + "security": "confidential" + }, + { + "firstName": "Bon", + "lastName": "FOURNIER", + "phone": "04 41 82 61 98", + "email": "zmartinez53@chevalier.org", + "country": "France", + "region": "Corse", + "employeeNumber": "79452-6559", + "security": "confidential" + }, + { + "firstName": "Artémis", + "lastName": "LECLERC", + "phone": "05 02 96 35 73", + "email": "cfontaine9@roux.name", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "38163-5687", + "security": "confidential" + }, + { + "firstName": "Parfait", + "lastName": "LAURENT", + "phone": "00 11 66 71 11", + "email": "ldurand9@legall.net", + "country": "France", + "region": "Normandie", + "employeeNumber": "40795", + "security": "confidential" + }, + { + "firstName": "Frédérique", + "lastName": "ROLLAND", + "phone": "01 00 25 40 24", + "email": "oriandevidal@sanchez.net", + "country": "France", + "region": "Pays de la Loire", + "employeeNumber": "09761", + "security": "confidential" + }, + { + "firstName": "Marcelin", + "lastName": "POIRIER", + "phone": "09 09 04 64 50", + "email": "spicard72@leclerc.fr", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "53747", + "security": "confidential" + }, + { + "firstName": "Aglaé", + "lastName": "LECLERC", + "phone": "05 15 64 87 11", + "email": "lucilledupont@martinez.name", + "country": "France", + "region": "Pays de la Loire", + "employeeNumber": "85981-9657", + "security": "confidential" + }, + { + "firstName": "Faustine", + "lastName": "CHEVALIER", + "phone": "04 08 94 23 71", + "email": "nbrun@mathieu.eu", + "country": "France", + "region": "Normandie", + "employeeNumber": "98827", + "security": "confidential" + }, + { + "firstName": "Adalbert", + "lastName": "PERRIN", + "phone": "02 50 61 67 28", + "email": "cyriaquelefebvre@marchal.name", + "country": "France", + "region": "Bourgogne-Franche-Comté", + "employeeNumber": "28664", + "security": "confidential" + }, + { + "firstName": "Nicéphore", + "lastName": "DUPONT", + "phone": "03 52 68 20 61", + "email": "vincianeperrin@lemoine.name", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "95436", + "security": "confidential" + }, + { + "firstName": "Jeanne", + "lastName": "MATHIEU", + "phone": "02 09 11 96 63", + "email": "percevalmeyer@schmitt.info", + "country": "France", + "region": "Bretagne", + "employeeNumber": "93396-3968", + "security": "confidential" + }, + { + "firstName": "Argine", + "lastName": "GUYOT", + "phone": "09 17 67 45 69", + "email": "yvasseur@henry.fr", + "country": "France", + "region": "Pays de la Loire", + "employeeNumber": "75734", + "security": "confidential" + }, + { + "firstName": "Fortuné", + "lastName": "ROLLAND", + "phone": "03 95 97 10 08", + "email": "icolin4@dupuis.info", + "country": "France", + "region": "Occitanie", + "employeeNumber": "93796-3575", + "security": "confidential" + }, + { + "firstName": "Audeline", + "lastName": "HUBERT", + "phone": "09 07 08 05 41", + "email": "xfontaine5@leroux.net", + "country": "France", + "region": "Bourgogne-Franche-Comté", + "employeeNumber": "84096-6349", + "security": "confidential" + }, + { + "firstName": "Simon", + "lastName": "GAUTHIER", + "phone": "02 42 25 69 81", + "email": "kleroy84@bertrand.com", + "country": "France", + "region": "Normandie", + "employeeNumber": "75281-6928", + "security": "confidential" + }, + { + "firstName": "Antoinette", + "lastName": "GONZALEZ", + "phone": "04 52 05 07 58", + "email": "edupont39@renault.fr", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "28878-9863", + "security": "confidential" + }, + { + "firstName": "Honorine", + "lastName": "SCHMITT", + "phone": "07 19 87 94 40", + "email": "tonninpons@vidal.net", + "country": "France", + "region": "Auvergne-Rhône-Alpes", + "employeeNumber": "83165", + "security": "confidential" + }, + { + "firstName": "Leufroy", + "lastName": "FABRE", + "phone": "02 87 47 18 22", + "email": "bsanchez89@rousseau.eu", + "country": "France", + "region": "Centre-Val de Loire", + "employeeNumber": "37323-0449", + "security": "confidential" + }, + { + "firstName": "Lorraine", + "lastName": "PONS", + "phone": "05 95 32 18 42", + "email": "mlucas7@colin.org", + "country": "France", + "region": "Bretagne", + "employeeNumber": "41311", + "security": "confidential" + }, + { + "firstName": "Mégane", + "lastName": "BARON", + "phone": "01 27 28 18 04", + "email": "gillesmartin@roux.net", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "86966-4193", + "security": "confidential" + }, + { + "firstName": "Adonis", + "lastName": "ADAM", + "phone": "02 44 53 37 42", + "email": "jgirard2@maillard.net", + "country": "France", + "region": "Normandie", + "employeeNumber": "04340-4850", + "security": "confidential" + }, + { + "firstName": "Aline", + "lastName": "FAURE", + "phone": "03 73 74 62 58", + "email": "egonzalez8@meunier.info", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "40164", + "security": "confidential" + }, + { + "firstName": "Ombline", + "lastName": "BRUN", + "phone": "06 69 32 25 39", + "email": "jocelyneguillot@dupont.info", + "country": "France", + "region": "Pays de la Loire", + "employeeNumber": "74670-3571", + "security": "confidential" + }, + { + "firstName": "Constantin", + "lastName": "MICHEL", + "phone": "09 85 60 34 33", + "email": "aroux@gautier.name", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "62411", + "security": "confidential" + }, + { + "firstName": "Odette", + "lastName": "GIRAUD", + "phone": "00 58 89 69 14", + "email": "croche@faure.name", + "country": "France", + "region": "Pays de la Loire", + "employeeNumber": "85824", + "security": "confidential" + }, + { + "firstName": "Hervé", + "lastName": "ROUSSEAU", + "phone": "09 45 87 18 38", + "email": "mleclerc@nguyen.org", + "country": "France", + "region": "Centre-Val de Loire", + "employeeNumber": "94657-5071", + "security": "confidential" + }, + { + "firstName": "Théodora", + "lastName": "RENARD", + "phone": "06 65 85 73 61", + "email": "faubry@martinez.org", + "country": "France", + "region": "Occitanie", + "employeeNumber": "71931-4945", + "security": "confidential" + }, + { + "firstName": "Fantin", + "lastName": "LAINE", + "phone": "00 17 36 64 16", + "email": "tlemoine2@henry.net", + "country": "France", + "region": "Bourgogne-Franche-Comté", + "employeeNumber": "83555", + "security": "confidential" + }, + { + "firstName": "Émile", + "lastName": "ROY", + "phone": "09 68 76 57 56", + "email": "lguyot5@schneider.org", + "country": "France", + "region": "Corse", + "employeeNumber": "28743", + "security": "confidential" + }, + { + "firstName": "Alexandrine", + "lastName": "PEREZ", + "phone": "01 27 44 33 99", + "email": "xjoly17@benoit.net", + "country": "France", + "region": "Bretagne", + "employeeNumber": "08327-5450", + "security": "confidential" + }, + { + "firstName": "Garnier", + "lastName": "BLANC", + "phone": "01 58 12 91 61", + "email": "pgauthier@henry.info", + "country": "France", + "region": "Grand Est", + "employeeNumber": "09375-4926", + "security": "confidential" + }, + { + "firstName": "Audeline", + "lastName": "BOURGEOIS", + "phone": "04 86 93 43 55", + "email": "amandinecarre@gerard.org", + "country": "France", + "region": "Grand Est", + "employeeNumber": "64492-5821", + "security": "confidential" + }, + { + "firstName": "Basilisse", + "lastName": "MARCHAL", + "phone": "00 64 92 82 66", + "email": "xroger0@dupuy.net", + "country": "France", + "region": "Centre-Val de Loire", + "employeeNumber": "50466-1244", + "security": "confidential" + }, + { + "firstName": "Alcine", + "lastName": "MARTY", + "phone": "02 84 11 45 31", + "email": "aparis@paris.com", + "country": "France", + "region": "Nouvelle-Aquitaine", + "employeeNumber": "87513", + "security": "confidential" + }, + { + "firstName": "Jourdain", + "lastName": "SANCHEZ", + "phone": "07 01 43 00 80", + "email": "xlecomte@vasseur.org", + "country": "France", + "region": "Bourgogne-Franche-Comté", + "employeeNumber": "45392-5563", + "security": "confidential" + }, + { + "firstName": "Jeanne", + "lastName": "GUERIN", + "phone": "04 07 24 75 62", + "email": "aliceleroux@laurent.com", + "country": "France", + "region": "Occitanie", + "employeeNumber": "59472-0564", + "security": "confidential" + }, + { + "firstName": "Camille", + "lastName": "DUMONT", + "phone": "07 65 40 06 70", + "email": "vjacquet@boyer.eu", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "26229-2057", + "security": "confidential" + }, + { + "firstName": "Adélaïde", + "lastName": "NICOLAS", + "phone": "08 79 36 69 17", + "email": "herbertjulien@duval.fr", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "40606-9992", + "security": "confidential" + }, + { + "firstName": "Axel", + "lastName": "SIMON", + "phone": "08 96 59 97 09", + "email": "edmeebreton@leroy.name", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "85483-8057", + "security": "confidential" + }, + { + "firstName": "Abigaelle", + "lastName": "BRUNET", + "phone": "09 57 49 04 95", + "email": "jlambert8@roche.info", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "63599", + "security": "confidential" + }, + { + "firstName": "Alix", + "lastName": "PAUL", + "phone": "03 40 29 52 06", + "email": "xjean@adam.name", + "country": "France", + "region": "Pays de la Loire", + "employeeNumber": "03636-3216", + "security": "confidential" + }, + { + "firstName": "Nine", + "lastName": "GAUTHIER", + "phone": "09 54 22 15 84", + "email": "qguillaume@lefevre.net", + "country": "France", + "region": "Bretagne", + "employeeNumber": "49995-2754", + "security": "confidential" + }, + { + "firstName": "Aimable", + "lastName": "CHARLES", + "phone": "06 95 49 35 39", + "email": "uperez52@legrand.name", + "country": "France", + "region": "Bretagne", + "employeeNumber": "52701", + "security": "confidential" + }, + { + "firstName": "Caroline", + "lastName": "NOEL", + "phone": "08 80 38 73 71", + "email": "pduval@roux.name", + "country": "France", + "region": "Bourgogne-Franche-Comté", + "employeeNumber": "17670-6405", + "security": "confidential" + }, + { + "firstName": "Aube", + "lastName": "VASSEUR", + "phone": "04 91 69 34 49", + "email": "zaubert44@deschamps.org", + "country": "France", + "region": "Occitanie", + "employeeNumber": "78786-9452", + "security": "confidential" + }, + { + "firstName": "Laurène", + "lastName": "MEYER", + "phone": "05 67 29 24 16", + "email": "dorinelambert@moreau.org", + "country": "France", + "region": "Corse", + "employeeNumber": "98802-1336", + "security": "confidential" + }, + { + "firstName": "Adel", + "lastName": "COLIN", + "phone": "09 40 90 53 85", + "email": "imarty36@philippe.name", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "64816", + "security": "confidential" + }, + { + "firstName": "Léna", + "lastName": "NGUYEN", + "phone": "04 83 66 79 78", + "email": "nmarchal41@berger.fr", + "country": "France", + "region": "Provence-Alpes-Côte d'Azur", + "employeeNumber": "72736", + "security": "confidential" + }, + { + "firstName": "Laura", + "lastName": "MARCHAND", + "phone": "06 92 14 62 70", + "email": "ngarcia0@dupuy.eu", + "country": "France", + "region": "Provence-Alpes-Côte d'Azur", + "employeeNumber": "41535-6744", + "security": "confidential" + }, + { + "firstName": "Aldric", + "lastName": "ROUSSEAU", + "phone": "01 46 99 17 34", + "email": "giselepoirier@perrin.name", + "country": "France", + "region": "Normandie", + "employeeNumber": "47090-0690", + "security": "confidential" + }, + { + "firstName": "Athalie", + "lastName": "CARRE", + "phone": "07 13 51 08 24", + "email": "amarie1@legrand.fr", + "country": "France", + "region": "Grand Est", + "employeeNumber": "33376", + "security": "confidential" + }, + { + "firstName": "Florian", + "lastName": "SCHNEIDER", + "phone": "07 64 23 90 57", + "email": "qmuller@henry.net", + "country": "France", + "region": "Grand Est", + "employeeNumber": "09627", + "security": "confidential" + }, + { + "firstName": "Cyriaque", + "lastName": "GAILLARD", + "phone": "09 97 77 30 59", + "email": "dlefebvre34@benoit.info", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "52842", + "security": "confidential" + }, + { + "firstName": "Hippolyte", + "lastName": "FRANCOIS", + "phone": "01 07 26 20 37", + "email": "jmarchand8@poirier.eu", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "66877", + "security": "confidential" + }, + { + "firstName": "Annonciade", + "lastName": "DESCHAMPS", + "phone": "03 38 19 74 62", + "email": "ochevalier4@leroux.eu", + "country": "France", + "region": "Provence-Alpes-Côte d'Azur", + "employeeNumber": "25239", + "security": "confidential" + }, + { + "firstName": "Adonise", + "lastName": "GUERIN", + "phone": "05 59 17 83 77", + "email": "celestinesanchez@garnier.net", + "country": "France", + "region": "Corse", + "employeeNumber": "28055-9019", + "security": "confidential" + }, + { + "firstName": "Conception", + "lastName": "REY", + "phone": "00 38 13 17 19", + "email": "jrobert@denis.name", + "country": "France", + "region": "Normandie", + "employeeNumber": "00502", + "security": "confidential" + }, + { + "firstName": "Angadrême", + "lastName": "VINCENT", + "phone": "03 79 86 01 72", + "email": "marceaurichard@durand.eu", + "country": "France", + "region": "Corse", + "employeeNumber": "24186-1062", + "security": "confidential" + }, + { + "firstName": "Delphine", + "lastName": "LE ROUX", + "phone": "04 46 70 39 89", + "email": "wmathieu@renault.fr", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "47263-2382", + "security": "confidential" + }, + { + "firstName": "Gabin", + "lastName": "BOURGEOIS", + "phone": "01 25 52 09 20", + "email": "wcarre22@mathieu.fr", + "country": "France", + "region": "Centre-Val de Loire", + "employeeNumber": "97467-1718", + "security": "confidential" + }, + { + "firstName": "Bérangère", + "lastName": "SCHMITT", + "phone": "01 78 44 09 27", + "email": "ndumont58@clement.eu", + "country": "France", + "region": "Corse", + "employeeNumber": "14655", + "security": "confidential" + }, + { + "firstName": "Arnould", + "lastName": "AUBRY", + "phone": "00 55 12 43 67", + "email": "qduval7@vasseur.info", + "country": "France", + "region": "Provence-Alpes-Côte d'Azur", + "employeeNumber": "95328", + "security": "confidential" + }, + { + "firstName": "Daniel", + "lastName": "VASSEUR", + "phone": "07 60 59 40 06", + "email": "fpierre@andre.com", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "28404-8895", + "security": "confidential" + }, + { + "firstName": "Paulette", + "lastName": "CHEVALIER", + "phone": "05 76 47 98 16", + "email": "deboraguillot@dupuy.net", + "country": "France", + "region": "Auvergne-Rhône-Alpes", + "employeeNumber": "82414-9145", + "security": "confidential" + }, + { + "firstName": "Sidoine", + "lastName": "CARPENTIER", + "phone": "06 13 52 26 40", + "email": "qrenard90@fournier.com", + "country": "France", + "region": "Bretagne", + "employeeNumber": "14011", + "security": "confidential" + }, + { + "firstName": "Béranger", + "lastName": "RODRIGUEZ", + "phone": "07 09 43 00 75", + "email": "taurinnicolas@gerard.org", + "country": "France", + "region": "Grand Est", + "employeeNumber": "61782-1334", + "security": "confidential" + }, + { + "firstName": "Jocelyn", + "lastName": "CARON", + "phone": "02 30 27 26 30", + "email": "kriviere@nicolas.org", + "country": "France", + "region": "Provence-Alpes-Côte d'Azur", + "employeeNumber": "72175", + "security": "confidential" + }, + { + "firstName": "Capucine", + "lastName": "HUBERT", + "phone": "07 24 82 39 79", + "email": "pvasseur2@breton.info", + "country": "France", + "region": "Occitanie", + "employeeNumber": "95394-9443", + "security": "confidential" + }, + { + "firstName": "Aaron", + "lastName": "LECOMTE", + "phone": "09 00 41 25 99", + "email": "trolland9@moreau.eu", + "country": "France", + "region": "Bourgogne-Franche-Comté", + "employeeNumber": "30036", + "security": "confidential" + }, + { + "firstName": "Lazare", + "lastName": "GARCIA", + "phone": "08 42 38 09 23", + "email": "ofontaine9@leclerc.org", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "56617", + "security": "confidential" + }, + { + "firstName": "Alcidie", + "lastName": "CARON", + "phone": "09 83 12 87 61", + "email": "qbonnet8@renard.info", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "94543-9283", + "security": "confidential" + }, + { + "firstName": "Honorine", + "lastName": "COLLET", + "phone": "08 70 45 41 78", + "email": "qbourgeois77@barbier.org", + "country": "France", + "region": "Bourgogne-Franche-Comté", + "employeeNumber": "59761-5344", + "security": "confidential" + }, + { + "firstName": "Joséphine", + "lastName": "LECOMTE", + "phone": "06 83 15 68 98", + "email": "eliafernandez@david.fr", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "06656-3067", + "security": "confidential" + }, + { + "firstName": "Brigitte", + "lastName": "BLANCHARD", + "phone": "01 13 21 70 78", + "email": "zmorel2@noel.com", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "92033-1976", + "security": "confidential" + }, + { + "firstName": "Rodolphe", + "lastName": "BRUN", + "phone": "05 86 79 82 12", + "email": "mpaul83@collet.net", + "country": "France", + "region": "Grand Est", + "employeeNumber": "76431-4612", + "security": "confidential" + }, + { + "firstName": "Clémentine", + "lastName": "VASSEUR", + "phone": "07 70 37 10 76", + "email": "uschneider@schneider.fr", + "country": "France", + "region": "Hauts-de-France", + "employeeNumber": "19780", + "security": "confidential" + }, + { + "firstName": "Amiel", + "lastName": "BRUN", + "phone": "09 45 85 13 17", + "email": "sguillot@bonnet.fr", + "country": "France", + "region": "Nouvelle-Aquitaine", + "employeeNumber": "39043-3916", + "security": "confidential" + }, + { + "firstName": "Macaire", + "lastName": "MARCHAL", + "phone": "03 77 55 25 49", + "email": "artemisaubry@prevost.eu", + "country": "France", + "region": "Bourgogne-Franche-Comté", + "employeeNumber": "44275", + "security": "confidential" + }, + { + "firstName": "Calixte", + "lastName": "MOREL", + "phone": "08 97 35 42 71", + "email": "uprevost14@caron.info", + "country": "France", + "region": "Île-de-France", + "employeeNumber": "72517-4062", + "security": "confidential" + }, + { + "firstName": "Jérôme", + "lastName": "GERARD", + "phone": "00 82 99 69 92", + "email": "brodriguez@cousin.info", + "country": "France", + "region": "Nouvelle-Aquitaine", + "employeeNumber": "02686-6469", + "security": "confidential" + }, + { + "firstName": "Théodora", + "lastName": "BONNET", + "phone": "03 93 02 43 95", + "email": "obourgeois@dufour.com", + "country": "France", + "region": "Provence-Alpes-Côte d'Azur", + "employeeNumber": "88665-7198", + "security": "confidential" + } +] \ No newline at end of file diff --git a/src/upsert.rs b/src/upsert.rs index 32801c43..4c4e8af0 100644 --- a/src/upsert.rs +++ b/src/upsert.rs @@ -10,9 +10,7 @@ use crate::{ error::{CallbackError, Error}, keys::KeyingMaterial, parameters::check_parameter_constraints, - structs::{ - BlockType, EncryptedTable, IndexedValue, Keyword, KeywordHash, Label, Uid, UpsertData, - }, + structs::{BlockType, EncryptedTable, IndexedValue, Keyword, Label, Uid, UpsertData}, FindexCallbacks, Uids, ENTRY_TABLE_KEY_DERIVATION_INFO, }; @@ -34,12 +32,15 @@ pub trait FindexUpsert< /// - `master_key` : Findex master key /// - `label` : additional public information used in key hashing /// - `items` : set of keywords used to index values + /// + /// Returns a map of keywords to booleans indicating whether the keyword + /// was already present in the database. async fn add( &mut self, master_key: &KeyingMaterial, label: &Label, items: HashMap>, - ) -> Result<(), Error> { + ) -> Result, Error> { self.upsert(master_key, label, items, HashMap::new()).await } @@ -50,12 +51,15 @@ pub trait FindexUpsert< /// - `master_key` : Findex master key /// - `label` : additional public information used in key hashing /// - `items` : set of keywords used to index values + /// + /// Returns a map of keywords to booleans indicating whether the keyword + /// was already present in the database. async fn remove( &mut self, master_key: &KeyingMaterial, label: &Label, items: HashMap>, - ) -> Result<(), Error> { + ) -> Result, Error> { self.upsert(master_key, label, HashMap::new(), items).await } @@ -68,27 +72,32 @@ pub trait FindexUpsert< /// - `additions` : values to indexed for a set of keywords /// - `deletions` : values to remove from the indexes for a set of /// keywords + /// + /// Returns a map of keywords to booleans indicating whether the keyword + /// was already present in the database. async fn upsert( &mut self, master_key: &KeyingMaterial, label: &Label, additions: HashMap>, deletions: HashMap>, - ) -> Result<(), Error> { + ) -> Result, Error> { check_parameter_constraints::(); let mut rng = CsRng::from_entropy(); let k_uid = master_key.derive_kmac_key::(ENTRY_TABLE_KEY_DERIVATION_INFO); let k_value = master_key.derive_dem_key(ENTRY_TABLE_KEY_DERIVATION_INFO); - let mut new_chains = - HashMap::>::with_capacity( - additions.len() + deletions.len(), - ); + // a reverse map of Keyword UIDs (e.g. hashes) to Keywords + let mut uid_to_keyword = HashMap::, Keyword>::new(); + + let mut new_chains = HashMap::>::with_capacity( + additions.len() + deletions.len(), + ); for (indexed_value, keywords) in additions { for keyword in keywords { new_chains - .entry(keyword.hash()) + .entry(keyword) .or_default() .insert(indexed_value.clone(), BlockType::Addition); } @@ -96,7 +105,7 @@ pub trait FindexUpsert< for (indexed_value, keywords) in deletions { for keyword in keywords { new_chains - .entry(keyword.hash()) + .entry(keyword) .or_default() .insert(indexed_value.clone(), BlockType::Deletion); } @@ -104,24 +113,38 @@ pub trait FindexUpsert< // Compute the Entry Table UIDs. let mut new_chains = new_chains .into_iter() - .map(|(keyword_hash, indexed_values)| { - ( - EntryTable::::generate_uid::( - &k_uid, - &keyword_hash, - label, - ), - (keyword_hash, indexed_values), - ) + .map(|(keyword, indexed_values)| { + let keyword_hash = keyword.hash(); + let uid = EntryTable::::generate_uid::( + &k_uid, + &keyword_hash, + label, + ); + uid_to_keyword.insert(uid, keyword); + (uid, (keyword_hash, indexed_values)) }) .collect::>(); // Query the Entry Table for these UIDs. - let mut encrypted_entry_table = self + let mut encrypted_entry_table: EncryptedTable = self .fetch_entry_table(Uids(new_chains.keys().copied().collect())) .await? .try_into()?; + // compute the map of keywords to booleans indicating whether the keyword + // was already present in the database + let mut keyword_presence = HashMap::new(); + for uid in encrypted_entry_table.keys() { + if let Some(keyword) = uid_to_keyword.remove(uid) { + keyword_presence.insert(keyword, true); + } + } + // whatever is left in `uid_to_keyword` is not in the database, + // i.e. not already present. Update the keyword_presence accordingly + for (_, keyword) in uid_to_keyword { + keyword_presence.insert(keyword, false); + } + while !new_chains.is_empty() { // Decrypt the Entry Table once and for all. let mut entry_table = @@ -148,7 +171,7 @@ pub trait FindexUpsert< new_chains.retain(|uid, _| encrypted_entry_table.contains_key(uid)); } - Ok(()) + Ok(keyword_presence) } /// Writes the given modifications to the indexes. Returns the current value diff --git a/tests/test_in_memory.rs b/tests/test_in_memory.rs index 3f18a9f4..7a7220ab 100644 --- a/tests/test_in_memory.rs +++ b/tests/test_in_memory.rs @@ -922,3 +922,93 @@ async fn test_search_cyclic_graph() { assert!(res_i.contains(&l_g)); assert!(res_i.contains(&l_h)); } + +#[actix_rt::test] +async fn test_keyword_presence() -> Result<(), Error> { + let mut rng = CsRng::from_entropy(); + + let label = Label::random(&mut rng); + + let master_key = KeyingMaterial::new(&mut rng); + + let mut indexed_value_to_keywords = HashMap::new(); + + // direct location robert doe + let robert_doe_location = Location::from("robert doe DB location"); + indexed_value_to_keywords.insert( + IndexedValue::Location(robert_doe_location.clone()), + hashset_keywords(&["robert", "doe"]), + ); + + // direct location john doe + let john_doe_location = Location::from("john doe DB location"); + indexed_value_to_keywords.insert( + IndexedValue::Location(john_doe_location.clone()), + hashset_keywords(&["john", "doe"]), + ); + + let mut findex = FindexInMemory::default(); + let presence = findex + .add(&master_key, &label, indexed_value_to_keywords) + .await?; + // the 3 keywords should not be present in the database + assert_eq!(presence.len(), 3); + assert!(presence.contains_key(&Keyword::from("robert"))); + assert!(presence.contains_key(&Keyword::from("doe"))); + assert!(presence.contains_key(&Keyword::from("john"))); + assert!(!*presence.get(&Keyword::from("robert")).unwrap()); + assert!(!*presence.get(&Keyword::from("doe")).unwrap()); + assert!(!*presence.get(&Keyword::from("john")).unwrap()); + + // Now insert a Robert Smith + let mut indexed_value_to_keywords = HashMap::new(); + let robert_smith_location = Location::from("robert smith DB location"); + indexed_value_to_keywords.insert( + IndexedValue::Location(robert_smith_location), + hashset_keywords(&["robert", "smith"]), + ); + let presence = findex + .add(&master_key, &label, indexed_value_to_keywords) + .await?; + // robert should be present, but not smith + assert_eq!(presence.len(), 2); + assert!(presence.contains_key(&Keyword::from("robert"))); + assert!(presence.contains_key(&Keyword::from("smith"))); + assert!(*presence.get(&Keyword::from("robert")).unwrap()); + assert!(!*presence.get(&Keyword::from("smith")).unwrap()); + + // Delete Robert Smith but add the Junior keyword + let robert_smith_location = Location::from("robert smith DB location"); + let mut indexed_value_to_keywords = HashMap::new(); + indexed_value_to_keywords.insert( + IndexedValue::Location(robert_smith_location.clone()), + hashset_keywords(&["robert", "smith", "junior"]), + ); + let presence = findex + .remove(&master_key, &label, indexed_value_to_keywords.clone()) + .await?; + // robert and smith should be present, but not junior + assert_eq!(presence.len(), 3); + assert!(presence.contains_key(&Keyword::from("robert"))); + assert!(presence.contains_key(&Keyword::from("smith"))); + assert!(presence.contains_key(&Keyword::from("junior"))); + assert!(*presence.get(&Keyword::from("robert")).unwrap()); + assert!(*presence.get(&Keyword::from("smith")).unwrap()); + assert!(!*presence.get(&Keyword::from("junior")).unwrap()); + + // however, the first delete create an entry for "junior, + // therefore deleting again will find it + let presence = findex + .remove(&master_key, &label, indexed_value_to_keywords.clone()) + .await?; + // robert and smith should be present, but not junior + assert_eq!(presence.len(), 3); + assert!(presence.contains_key(&Keyword::from("robert"))); + assert!(presence.contains_key(&Keyword::from("smith"))); + assert!(presence.contains_key(&Keyword::from("junior"))); + assert!(*presence.get(&Keyword::from("robert")).unwrap()); + assert!(*presence.get(&Keyword::from("smith")).unwrap()); + assert!(*presence.get(&Keyword::from("junior")).unwrap()); + + Ok(()) +} From 46d4eb2a78445e9946ffeaf8d1f53f21f1cb0fa6 Mon Sep 17 00:00:00 2001 From: Bruno Grieder Date: Tue, 1 Aug 2023 09:09:17 +0200 Subject: [PATCH 6/9] removed self mutability on all interfaces --- .vscode/settings.json | 3 ++ benches/benches.rs | 2 +- examples/search.rs | 2 +- examples/upsert.rs | 2 +- src/callbacks.rs | 6 +-- src/compact.rs | 2 +- src/in_memory_example.rs | 99 +++++++++++++++++++++++++++++++--------- src/search.rs | 6 +-- src/upsert.rs | 8 ++-- tests/test_in_memory.rs | 10 ++-- 10 files changed, 99 insertions(+), 41 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..4441301b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "rust-analyzer.cargo.features": ["in_memory"] +} diff --git a/benches/benches.rs b/benches/benches.rs index 77497535..514200ab 100644 --- a/benches/benches.rs +++ b/benches/benches.rs @@ -45,7 +45,7 @@ fn bench_search(c: &mut Criterion) { // // Prepare indexes to be search // - let mut findex = FindexInMemory::default(); + let findex = FindexInMemory::default(); block_on(findex.add(&master_key, &label, locations_and_words)).expect("msg"); println!("Entry Table length: {}", findex.entry_table_len()); diff --git a/examples/search.rs b/examples/search.rs index e11cb7ae..8c17cbde 100644 --- a/examples/search.rs +++ b/examples/search.rs @@ -44,7 +44,7 @@ fn main() { // // Prepare indexes to be search // - let mut findex = FindexInMemory::default(); + let findex = FindexInMemory::default(); block_on(findex.add(&master_key, &label, locations_and_words)).expect("msg"); // diff --git a/examples/upsert.rs b/examples/upsert.rs index 9b689f36..72b91174 100644 --- a/examples/upsert.rs +++ b/examples/upsert.rs @@ -55,7 +55,7 @@ fn main() { hashset_keywords(&["rob"]), ); - let mut findex = FindexInMemory::default(); + let findex = FindexInMemory::default(); for _ in 0..100_000 { block_on(findex.add(&master_key, &label, indexed_value_to_keywords.clone())).unwrap(); diff --git a/src/callbacks.rs b/src/callbacks.rs index 3d7c53e7..a89c8235 100644 --- a/src/callbacks.rs +++ b/src/callbacks.rs @@ -79,7 +79,7 @@ pub trait FindexCallbacks, ) -> Result, Error>; @@ -93,7 +93,7 @@ pub trait FindexCallbacks) -> Result<(), Error>; + async fn insert_chain_table(&self, items: EncryptedTable) -> Result<(), Error>; /// Updates the indexes with the given data. /// @@ -144,7 +144,7 @@ pub trait FindexCallbacks, new_entry_table_items: EncryptedTable, new_chain_table_items: EncryptedTable, diff --git a/src/compact.rs b/src/compact.rs index 3d818b79..342f551b 100644 --- a/src/compact.rs +++ b/src/compact.rs @@ -54,7 +54,7 @@ pub trait FindexCompact< /// **WARNING**: the compact operation *cannot* be done concurrently with /// upsert operations. This could result in corrupted indexes. async fn compact( - &mut self, + &self, master_key: &KeyingMaterial, new_master_key: &KeyingMaterial, label: &Label, diff --git a/src/in_memory_example.rs b/src/in_memory_example.rs index 04216cd0..8098f521 100644 --- a/src/in_memory_example.rs +++ b/src/in_memory_example.rs @@ -1,6 +1,7 @@ use std::{ collections::{HashMap, HashSet}, fmt::Display, + sync::RwLock, }; use cosmian_crypto_core::{ @@ -31,10 +32,10 @@ impl std::error::Error for ExampleError {} impl crate::CallbackError for ExampleError {} -#[derive(Default, Clone)] +#[derive(Default)] pub struct FindexInMemory { - entry_table: EncryptedTable, - chain_table: EncryptedTable, + entry_table: RwLock>, + chain_table: RwLock>, removed_locations: HashSet, pub check_progress_callback_next_keyword: bool, pub progress_callback_cancel: bool, @@ -44,14 +45,22 @@ impl FindexInMemory { /// The entry table length (number of records) #[must_use] pub fn entry_table_len(&self) -> usize { - self.entry_table.len() + self.entry_table + .read() + .expect("entry table lock poisoned") + .len() } /// The entry table size in bytes #[must_use] pub fn entry_table_size(&self) -> usize { let mut size = 0; - for (k, v) in self.entry_table.iter() { + for (k, v) in self + .entry_table + .read() + .expect("entry table lock poisoned") + .iter() + { size += k.len() + v.len(); } size @@ -60,14 +69,22 @@ impl FindexInMemory { /// The chain table length (number of records) #[must_use] pub fn chain_table_len(&self) -> usize { - self.chain_table.len() + self.chain_table + .read() + .expect("chain table lock poisoned") + .len() } /// The entry table size in bytes #[must_use] pub fn chain_table_size(&self) -> usize { let mut size = 0; - for (k, v) in self.chain_table.iter() { + for (k, v) in self + .chain_table + .read() + .expect("chain table lock poisoned") + .iter() + { size += k.len() + v.len(); } size @@ -129,7 +146,14 @@ impl FindexCallbacks } async fn fetch_all_entry_table_uids(&self) -> Result, ExampleError> { - let uids = Uids(self.entry_table.keys().copied().collect()); + let uids = Uids( + self.entry_table + .read() + .expect("entry table lock poisoned") + .keys() + .copied() + .collect(), + ); Ok(uids) } @@ -141,6 +165,8 @@ impl FindexCallbacks .into_iter() .filter_map(|uid| { self.entry_table + .read() + .expect("entry table lock poisoned") .get(&uid) .cloned() .map(|value| (uid, value)) @@ -155,7 +181,12 @@ impl FindexCallbacks ) -> Result, ExampleError> { let mut items = EncryptedTable::with_capacity(chain_table_uids.len()); for uid in chain_table_uids { - if let Some(value) = self.chain_table.get(&uid) { + if let Some(value) = self + .chain_table + .read() + .expect("chain table lock poisoned") + .get(&uid) + { items.insert(uid, value.clone()); } } @@ -163,7 +194,7 @@ impl FindexCallbacks } async fn upsert_entry_table( - &mut self, + &self, modifications: UpsertData, ) -> Result, ExampleError> { let mut rng = CsRng::from_entropy(); @@ -171,47 +202,71 @@ impl FindexCallbacks // Simulate insertion failures. for (uid, (old_value, new_value)) in modifications { // Reject insert with probability 0.2. - if self.entry_table.contains_key(&uid) && rng.gen_range(0..5) == 0 { + if self + .entry_table + .read() + .expect("entry table lock poisoned") + .contains_key(&uid) + && rng.gen_range(0..5) == 0 + { rejected.insert(uid, old_value.unwrap_or_default()); } else { - self.entry_table.insert(uid, new_value); + self.entry_table + .write() + .expect("entry table lock poisoned") + .insert(uid, new_value); } } Ok(rejected) } async fn insert_chain_table( - &mut self, + &self, items: EncryptedTable, ) -> Result<(), ExampleError> { for (uid, value) in items { - if self.chain_table.contains_key(&uid) { + if self + .chain_table + .read() + .expect("entry table lock poisoned") + .contains_key(&uid) + { return Err(ExampleError(format!( "Conflict in Chain Table for UID: {uid:?}" ))); } - self.chain_table.insert(uid, value); + self.chain_table + .write() + .expect("chain table lock poisoned") + .insert(uid, value); } Ok(()) } async fn update_lines( - &mut self, + &self, chain_table_uids_to_remove: Uids, new_encrypted_entry_table_items: EncryptedTable, new_encrypted_chain_table_items: EncryptedTable, ) -> Result<(), ExampleError> { - self.entry_table = new_encrypted_entry_table_items; + *self.entry_table.write().expect("entry table lock poisoned") = + new_encrypted_entry_table_items; for new_encrypted_chain_table_item in new_encrypted_chain_table_items { - self.chain_table.insert( - new_encrypted_chain_table_item.0, - new_encrypted_chain_table_item.1, - ); + self.chain_table + .write() + .expect("chain table lock poisoned") + .insert( + new_encrypted_chain_table_item.0, + new_encrypted_chain_table_item.1, + ); } for removed_chain_table_uid in chain_table_uids_to_remove { - self.chain_table.remove(&removed_chain_table_uid); + self.chain_table + .write() + .expect("chain table lock poisoned") + .remove(&removed_chain_table_uid); } Ok(()) diff --git a/src/search.rs b/src/search.rs index 39cc8c97..c95b5b2b 100644 --- a/src/search.rs +++ b/src/search.rs @@ -36,7 +36,7 @@ pub trait FindexSearch< /// - `label` : public label /// - `keywords` : keywords to search async fn core_search( - &mut self, + &self, k_uid: &KmacKey, k_value: &DemKey, label: &Label, @@ -129,7 +129,7 @@ pub trait FindexSearch< /// - `label` : public label used for hashing /// - `keywords` : keywords to search using Findex async fn iterative_search( - &mut self, + &self, master_key: &KeyingMaterial, label: &Label, mut keywords: HashSet, @@ -178,7 +178,7 @@ pub trait FindexSearch< /// - `keywords` : keywords to search /// - `max_depth` : maximum recursion depth allowed async fn search( - &mut self, + &self, master_key: &KeyingMaterial, label: &Label, keywords: HashSet, diff --git a/src/upsert.rs b/src/upsert.rs index 4c4e8af0..b826f94e 100644 --- a/src/upsert.rs +++ b/src/upsert.rs @@ -36,7 +36,7 @@ pub trait FindexUpsert< /// Returns a map of keywords to booleans indicating whether the keyword /// was already present in the database. async fn add( - &mut self, + &self, master_key: &KeyingMaterial, label: &Label, items: HashMap>, @@ -55,7 +55,7 @@ pub trait FindexUpsert< /// Returns a map of keywords to booleans indicating whether the keyword /// was already present in the database. async fn remove( - &mut self, + &self, master_key: &KeyingMaterial, label: &Label, items: HashMap>, @@ -76,7 +76,7 @@ pub trait FindexUpsert< /// Returns a map of keywords to booleans indicating whether the keyword /// was already present in the database. async fn upsert( - &mut self, + &self, master_key: &KeyingMaterial, label: &Label, additions: HashMap>, @@ -182,7 +182,7 @@ pub trait FindexUpsert< /// - `new_entry_table` : new Entry Table /// - `chain_table_additions` : entries to be added to the Chain Table async fn write_indexes( - &mut self, + &self, old_entry_table: EncryptedTable, new_entry_table: EncryptedTable, mut chain_table_additions: HashMap, EncryptedTable>, diff --git a/tests/test_in_memory.rs b/tests/test_in_memory.rs index 7a7220ab..d5cbb633 100644 --- a/tests/test_in_memory.rs +++ b/tests/test_in_memory.rs @@ -464,8 +464,8 @@ async fn test_first_names() -> Result<(), Error> { const MAX_FIRST_NAMES: usize = 1000; let mut rng = rand::thread_rng(); let master_key = KeyingMaterial::new(&mut rng); - let mut graph_findex = FindexInMemory::default(); - let mut naive_findex = FindexInMemory::default(); + let graph_findex = FindexInMemory::default(); + let naive_findex = FindexInMemory::default(); // Keywords that will be searched later to run tests let mut searches: HashSet = HashSet::new(); @@ -595,7 +595,7 @@ async fn test_first_names() -> Result<(), Error> { async fn test_graph_compacting() { let mut rng = CsRng::from_entropy(); let mut master_key = KeyingMaterial::new(&mut rng); - let mut findex = FindexInMemory::default(); + let findex = FindexInMemory::default(); let mut indexed_value_to_keywords = HashMap::new(); let rob_keyword = Keyword::from(b"rob".to_vec()); @@ -780,7 +780,7 @@ async fn test_live_compacting() { #[actix_rt::test] async fn test_search_cyclic_graph() { let mut rng = CsRng::from_entropy(); - let mut findex = FindexInMemory::default(); + let findex = FindexInMemory::default(); let label = Label::random(&mut rng); let master_key = KeyingMaterial::new(&mut rng); @@ -947,7 +947,7 @@ async fn test_keyword_presence() -> Result<(), Error> { hashset_keywords(&["john", "doe"]), ); - let mut findex = FindexInMemory::default(); + let findex = FindexInMemory::default(); let presence = findex .add(&master_key, &label, indexed_value_to_keywords) .await?; From b958139d335b97e6e5ab1a48d19b678ac8fe9a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Tue, 1 Aug 2023 10:18:14 +0200 Subject: [PATCH 7/9] fix: review comments --- .DS_Store | Bin 6148 -> 0 bytes french_dataset.json | 1002 --------------------------------------- src/upsert.rs | 62 +-- tests/test_in_memory.rs | 147 +----- 4 files changed, 44 insertions(+), 1167 deletions(-) delete mode 100644 .DS_Store delete mode 100644 french_dataset.json diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 76f1144ad0c66984be340b470032996987b402c2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-O8-BN@e6nb3nS}@vDEM7vbFJMFuDm5WNgE3p0)E-J9XMG``#OHBl zcLSF8;7P>Jz~-BspWU4gvOkP5KAeTejI|hJ0vaMmWrLu3qpM1k=d*0+Pj9hz zrBo7>x*uG}(R|$6KUZn)$7wXx32_)f$lXnxhH5@kvoy?fu4f#gAsXXWd$Bm}4LY*l zb(S5u7@RmA+3WU~%Z4~OJUY7^KPNA#dNX8l;9bj}#S-2@S*z>So2H3MAHi4VmGOkc z05L!e5Chx8fIS05b9+jrN{Int;D-$0{ve?tx)uwAdh37>UZ2rlLqq``-x7$@qHD1* z2p$k_QUOgWw@(ai(!nlmoNKW#Xwn(iGs8G`=JN5v_3U7mI-GIWAbp7eVqlels_C}y z{J(%-X5%A&HHCV_05R~-7~r*$HyT1w_H11%56@Z&?ExAJ#ucc5fWC4GfDY~>ZS6FE bi8{o&77K$o3fgr#AYBABA@m^zeu056xwcIv diff --git a/french_dataset.json b/french_dataset.json deleted file mode 100644 index ebf9479f..00000000 --- a/french_dataset.json +++ /dev/null @@ -1,1002 +0,0 @@ -[ - { - "firstName": "Barthélemy", - "lastName": "MEUNIER", - "phone": "06 26 07 18 36", - "email": "tlaine87@dumont.fr", - "country": "France", - "region": "Grand Est", - "employeeNumber": "24513-1783", - "security": "confidential" - }, - { - "firstName": "Aaron", - "lastName": "LECLERCQ", - "phone": "08 72 64 64 25", - "email": "jlaurent9@gautier.eu", - "country": "France", - "region": "Pays de la Loire", - "employeeNumber": "92480-3186", - "security": "confidential" - }, - { - "firstName": "Annabelle", - "lastName": "REY", - "phone": "09 32 88 69 26", - "email": "zguyot@nguyen.fr", - "country": "France", - "region": "Centre-Val de Loire", - "employeeNumber": "91301", - "security": "confidential" - }, - { - "firstName": "Nine", - "lastName": "GARNIER", - "phone": "02 01 46 40 62", - "email": "gonthierlemaire@paris.com", - "country": "France", - "region": "Occitanie", - "employeeNumber": "12601", - "security": "confidential" - }, - { - "firstName": "Médéric", - "lastName": "GAUTIER", - "phone": "01 57 90 71 41", - "email": "anaisleroux@paul.eu", - "country": "France", - "region": "Bretagne", - "employeeNumber": "69533-3597", - "security": "confidential" - }, - { - "firstName": "Jacqueline", - "lastName": "MOULIN", - "phone": "06 80 74 10 53", - "email": "gleroux9@lefevre.net", - "country": "France", - "region": "Corse", - "employeeNumber": "22706", - "security": "confidential" - }, - { - "firstName": "Mathurin", - "lastName": "POIRIER", - "phone": "04 93 14 57 35", - "email": "grobin57@fernandez.com", - "country": "France", - "region": "Grand Est", - "employeeNumber": "76458", - "security": "confidential" - }, - { - "firstName": "Émilie", - "lastName": "BARRE", - "phone": "07 40 48 65 75", - "email": "cmeunier8@olivier.fr", - "country": "France", - "region": "Grand Est", - "employeeNumber": "72777-7930", - "security": "confidential" - }, - { - "firstName": "Noël", - "lastName": "MAILLARD", - "phone": "05 08 92 26 61", - "email": "edumont45@lefebvre.com", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "22914", - "security": "confidential" - }, - { - "firstName": "Bernadette", - "lastName": "DUVAL", - "phone": "09 86 41 24 01", - "email": "ylouis@cousin.org", - "country": "France", - "region": "Normandie", - "employeeNumber": "91526", - "security": "confidential" - }, - { - "firstName": "Aliénor", - "lastName": "FRANCOIS", - "phone": "01 58 39 36 80", - "email": "nlaurent@roche.com", - "country": "France", - "region": "Bretagne", - "employeeNumber": "08276", - "security": "confidential" - }, - { - "firstName": "Alcime", - "lastName": "MARTY", - "phone": "01 80 74 08 46", - "email": "bguillot1@schneider.info", - "country": "France", - "region": "Corse", - "employeeNumber": "51800", - "security": "confidential" - }, - { - "firstName": "Ismérie", - "lastName": "BERGER", - "phone": "09 51 90 51 33", - "email": "rbourgeois8@guillot.eu", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "44218-8208", - "security": "confidential" - }, - { - "firstName": "César", - "lastName": "BLANC", - "phone": "07 14 51 93 93", - "email": "kboyer2@lefevre.com", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "22436", - "security": "confidential" - }, - { - "firstName": "Julia", - "lastName": "RODRIGUEZ", - "phone": "07 89 77 50 12", - "email": "opierre0@colin.eu", - "country": "France", - "region": "Pays de la Loire", - "employeeNumber": "32837", - "security": "confidential" - }, - { - "firstName": "Angilran", - "lastName": "BLANCHARD", - "phone": "07 03 17 37 73", - "email": "rroger50@aubert.net", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "19262", - "security": "confidential" - }, - { - "firstName": "Clélie", - "lastName": "GUILLAUME", - "phone": "03 07 85 19 62", - "email": "kbrun@laurent.info", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "70688", - "security": "confidential" - }, - { - "firstName": "Clarisse", - "lastName": "LECOMTE", - "phone": "09 25 76 37 60", - "email": "ylambert9@sanchez.info", - "country": "France", - "region": "Grand Est", - "employeeNumber": "99208", - "security": "confidential" - }, - { - "firstName": "Henriette", - "lastName": "LEROUX", - "phone": "00 39 00 63 65", - "email": "qjacquet66@dumas.info", - "country": "France", - "region": "Auvergne-Rhône-Alpes", - "employeeNumber": "72500-2982", - "security": "confidential" - }, - { - "firstName": "Laurine", - "lastName": "LECLERCQ", - "phone": "07 46 90 69 09", - "email": "wleclerc@dubois.org", - "country": "France", - "region": "Bourgogne-Franche-Comté", - "employeeNumber": "43573", - "security": "confidential" - }, - { - "firstName": "Bon", - "lastName": "FOURNIER", - "phone": "04 41 82 61 98", - "email": "zmartinez53@chevalier.org", - "country": "France", - "region": "Corse", - "employeeNumber": "79452-6559", - "security": "confidential" - }, - { - "firstName": "Artémis", - "lastName": "LECLERC", - "phone": "05 02 96 35 73", - "email": "cfontaine9@roux.name", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "38163-5687", - "security": "confidential" - }, - { - "firstName": "Parfait", - "lastName": "LAURENT", - "phone": "00 11 66 71 11", - "email": "ldurand9@legall.net", - "country": "France", - "region": "Normandie", - "employeeNumber": "40795", - "security": "confidential" - }, - { - "firstName": "Frédérique", - "lastName": "ROLLAND", - "phone": "01 00 25 40 24", - "email": "oriandevidal@sanchez.net", - "country": "France", - "region": "Pays de la Loire", - "employeeNumber": "09761", - "security": "confidential" - }, - { - "firstName": "Marcelin", - "lastName": "POIRIER", - "phone": "09 09 04 64 50", - "email": "spicard72@leclerc.fr", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "53747", - "security": "confidential" - }, - { - "firstName": "Aglaé", - "lastName": "LECLERC", - "phone": "05 15 64 87 11", - "email": "lucilledupont@martinez.name", - "country": "France", - "region": "Pays de la Loire", - "employeeNumber": "85981-9657", - "security": "confidential" - }, - { - "firstName": "Faustine", - "lastName": "CHEVALIER", - "phone": "04 08 94 23 71", - "email": "nbrun@mathieu.eu", - "country": "France", - "region": "Normandie", - "employeeNumber": "98827", - "security": "confidential" - }, - { - "firstName": "Adalbert", - "lastName": "PERRIN", - "phone": "02 50 61 67 28", - "email": "cyriaquelefebvre@marchal.name", - "country": "France", - "region": "Bourgogne-Franche-Comté", - "employeeNumber": "28664", - "security": "confidential" - }, - { - "firstName": "Nicéphore", - "lastName": "DUPONT", - "phone": "03 52 68 20 61", - "email": "vincianeperrin@lemoine.name", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "95436", - "security": "confidential" - }, - { - "firstName": "Jeanne", - "lastName": "MATHIEU", - "phone": "02 09 11 96 63", - "email": "percevalmeyer@schmitt.info", - "country": "France", - "region": "Bretagne", - "employeeNumber": "93396-3968", - "security": "confidential" - }, - { - "firstName": "Argine", - "lastName": "GUYOT", - "phone": "09 17 67 45 69", - "email": "yvasseur@henry.fr", - "country": "France", - "region": "Pays de la Loire", - "employeeNumber": "75734", - "security": "confidential" - }, - { - "firstName": "Fortuné", - "lastName": "ROLLAND", - "phone": "03 95 97 10 08", - "email": "icolin4@dupuis.info", - "country": "France", - "region": "Occitanie", - "employeeNumber": "93796-3575", - "security": "confidential" - }, - { - "firstName": "Audeline", - "lastName": "HUBERT", - "phone": "09 07 08 05 41", - "email": "xfontaine5@leroux.net", - "country": "France", - "region": "Bourgogne-Franche-Comté", - "employeeNumber": "84096-6349", - "security": "confidential" - }, - { - "firstName": "Simon", - "lastName": "GAUTHIER", - "phone": "02 42 25 69 81", - "email": "kleroy84@bertrand.com", - "country": "France", - "region": "Normandie", - "employeeNumber": "75281-6928", - "security": "confidential" - }, - { - "firstName": "Antoinette", - "lastName": "GONZALEZ", - "phone": "04 52 05 07 58", - "email": "edupont39@renault.fr", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "28878-9863", - "security": "confidential" - }, - { - "firstName": "Honorine", - "lastName": "SCHMITT", - "phone": "07 19 87 94 40", - "email": "tonninpons@vidal.net", - "country": "France", - "region": "Auvergne-Rhône-Alpes", - "employeeNumber": "83165", - "security": "confidential" - }, - { - "firstName": "Leufroy", - "lastName": "FABRE", - "phone": "02 87 47 18 22", - "email": "bsanchez89@rousseau.eu", - "country": "France", - "region": "Centre-Val de Loire", - "employeeNumber": "37323-0449", - "security": "confidential" - }, - { - "firstName": "Lorraine", - "lastName": "PONS", - "phone": "05 95 32 18 42", - "email": "mlucas7@colin.org", - "country": "France", - "region": "Bretagne", - "employeeNumber": "41311", - "security": "confidential" - }, - { - "firstName": "Mégane", - "lastName": "BARON", - "phone": "01 27 28 18 04", - "email": "gillesmartin@roux.net", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "86966-4193", - "security": "confidential" - }, - { - "firstName": "Adonis", - "lastName": "ADAM", - "phone": "02 44 53 37 42", - "email": "jgirard2@maillard.net", - "country": "France", - "region": "Normandie", - "employeeNumber": "04340-4850", - "security": "confidential" - }, - { - "firstName": "Aline", - "lastName": "FAURE", - "phone": "03 73 74 62 58", - "email": "egonzalez8@meunier.info", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "40164", - "security": "confidential" - }, - { - "firstName": "Ombline", - "lastName": "BRUN", - "phone": "06 69 32 25 39", - "email": "jocelyneguillot@dupont.info", - "country": "France", - "region": "Pays de la Loire", - "employeeNumber": "74670-3571", - "security": "confidential" - }, - { - "firstName": "Constantin", - "lastName": "MICHEL", - "phone": "09 85 60 34 33", - "email": "aroux@gautier.name", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "62411", - "security": "confidential" - }, - { - "firstName": "Odette", - "lastName": "GIRAUD", - "phone": "00 58 89 69 14", - "email": "croche@faure.name", - "country": "France", - "region": "Pays de la Loire", - "employeeNumber": "85824", - "security": "confidential" - }, - { - "firstName": "Hervé", - "lastName": "ROUSSEAU", - "phone": "09 45 87 18 38", - "email": "mleclerc@nguyen.org", - "country": "France", - "region": "Centre-Val de Loire", - "employeeNumber": "94657-5071", - "security": "confidential" - }, - { - "firstName": "Théodora", - "lastName": "RENARD", - "phone": "06 65 85 73 61", - "email": "faubry@martinez.org", - "country": "France", - "region": "Occitanie", - "employeeNumber": "71931-4945", - "security": "confidential" - }, - { - "firstName": "Fantin", - "lastName": "LAINE", - "phone": "00 17 36 64 16", - "email": "tlemoine2@henry.net", - "country": "France", - "region": "Bourgogne-Franche-Comté", - "employeeNumber": "83555", - "security": "confidential" - }, - { - "firstName": "Émile", - "lastName": "ROY", - "phone": "09 68 76 57 56", - "email": "lguyot5@schneider.org", - "country": "France", - "region": "Corse", - "employeeNumber": "28743", - "security": "confidential" - }, - { - "firstName": "Alexandrine", - "lastName": "PEREZ", - "phone": "01 27 44 33 99", - "email": "xjoly17@benoit.net", - "country": "France", - "region": "Bretagne", - "employeeNumber": "08327-5450", - "security": "confidential" - }, - { - "firstName": "Garnier", - "lastName": "BLANC", - "phone": "01 58 12 91 61", - "email": "pgauthier@henry.info", - "country": "France", - "region": "Grand Est", - "employeeNumber": "09375-4926", - "security": "confidential" - }, - { - "firstName": "Audeline", - "lastName": "BOURGEOIS", - "phone": "04 86 93 43 55", - "email": "amandinecarre@gerard.org", - "country": "France", - "region": "Grand Est", - "employeeNumber": "64492-5821", - "security": "confidential" - }, - { - "firstName": "Basilisse", - "lastName": "MARCHAL", - "phone": "00 64 92 82 66", - "email": "xroger0@dupuy.net", - "country": "France", - "region": "Centre-Val de Loire", - "employeeNumber": "50466-1244", - "security": "confidential" - }, - { - "firstName": "Alcine", - "lastName": "MARTY", - "phone": "02 84 11 45 31", - "email": "aparis@paris.com", - "country": "France", - "region": "Nouvelle-Aquitaine", - "employeeNumber": "87513", - "security": "confidential" - }, - { - "firstName": "Jourdain", - "lastName": "SANCHEZ", - "phone": "07 01 43 00 80", - "email": "xlecomte@vasseur.org", - "country": "France", - "region": "Bourgogne-Franche-Comté", - "employeeNumber": "45392-5563", - "security": "confidential" - }, - { - "firstName": "Jeanne", - "lastName": "GUERIN", - "phone": "04 07 24 75 62", - "email": "aliceleroux@laurent.com", - "country": "France", - "region": "Occitanie", - "employeeNumber": "59472-0564", - "security": "confidential" - }, - { - "firstName": "Camille", - "lastName": "DUMONT", - "phone": "07 65 40 06 70", - "email": "vjacquet@boyer.eu", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "26229-2057", - "security": "confidential" - }, - { - "firstName": "Adélaïde", - "lastName": "NICOLAS", - "phone": "08 79 36 69 17", - "email": "herbertjulien@duval.fr", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "40606-9992", - "security": "confidential" - }, - { - "firstName": "Axel", - "lastName": "SIMON", - "phone": "08 96 59 97 09", - "email": "edmeebreton@leroy.name", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "85483-8057", - "security": "confidential" - }, - { - "firstName": "Abigaelle", - "lastName": "BRUNET", - "phone": "09 57 49 04 95", - "email": "jlambert8@roche.info", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "63599", - "security": "confidential" - }, - { - "firstName": "Alix", - "lastName": "PAUL", - "phone": "03 40 29 52 06", - "email": "xjean@adam.name", - "country": "France", - "region": "Pays de la Loire", - "employeeNumber": "03636-3216", - "security": "confidential" - }, - { - "firstName": "Nine", - "lastName": "GAUTHIER", - "phone": "09 54 22 15 84", - "email": "qguillaume@lefevre.net", - "country": "France", - "region": "Bretagne", - "employeeNumber": "49995-2754", - "security": "confidential" - }, - { - "firstName": "Aimable", - "lastName": "CHARLES", - "phone": "06 95 49 35 39", - "email": "uperez52@legrand.name", - "country": "France", - "region": "Bretagne", - "employeeNumber": "52701", - "security": "confidential" - }, - { - "firstName": "Caroline", - "lastName": "NOEL", - "phone": "08 80 38 73 71", - "email": "pduval@roux.name", - "country": "France", - "region": "Bourgogne-Franche-Comté", - "employeeNumber": "17670-6405", - "security": "confidential" - }, - { - "firstName": "Aube", - "lastName": "VASSEUR", - "phone": "04 91 69 34 49", - "email": "zaubert44@deschamps.org", - "country": "France", - "region": "Occitanie", - "employeeNumber": "78786-9452", - "security": "confidential" - }, - { - "firstName": "Laurène", - "lastName": "MEYER", - "phone": "05 67 29 24 16", - "email": "dorinelambert@moreau.org", - "country": "France", - "region": "Corse", - "employeeNumber": "98802-1336", - "security": "confidential" - }, - { - "firstName": "Adel", - "lastName": "COLIN", - "phone": "09 40 90 53 85", - "email": "imarty36@philippe.name", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "64816", - "security": "confidential" - }, - { - "firstName": "Léna", - "lastName": "NGUYEN", - "phone": "04 83 66 79 78", - "email": "nmarchal41@berger.fr", - "country": "France", - "region": "Provence-Alpes-Côte d'Azur", - "employeeNumber": "72736", - "security": "confidential" - }, - { - "firstName": "Laura", - "lastName": "MARCHAND", - "phone": "06 92 14 62 70", - "email": "ngarcia0@dupuy.eu", - "country": "France", - "region": "Provence-Alpes-Côte d'Azur", - "employeeNumber": "41535-6744", - "security": "confidential" - }, - { - "firstName": "Aldric", - "lastName": "ROUSSEAU", - "phone": "01 46 99 17 34", - "email": "giselepoirier@perrin.name", - "country": "France", - "region": "Normandie", - "employeeNumber": "47090-0690", - "security": "confidential" - }, - { - "firstName": "Athalie", - "lastName": "CARRE", - "phone": "07 13 51 08 24", - "email": "amarie1@legrand.fr", - "country": "France", - "region": "Grand Est", - "employeeNumber": "33376", - "security": "confidential" - }, - { - "firstName": "Florian", - "lastName": "SCHNEIDER", - "phone": "07 64 23 90 57", - "email": "qmuller@henry.net", - "country": "France", - "region": "Grand Est", - "employeeNumber": "09627", - "security": "confidential" - }, - { - "firstName": "Cyriaque", - "lastName": "GAILLARD", - "phone": "09 97 77 30 59", - "email": "dlefebvre34@benoit.info", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "52842", - "security": "confidential" - }, - { - "firstName": "Hippolyte", - "lastName": "FRANCOIS", - "phone": "01 07 26 20 37", - "email": "jmarchand8@poirier.eu", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "66877", - "security": "confidential" - }, - { - "firstName": "Annonciade", - "lastName": "DESCHAMPS", - "phone": "03 38 19 74 62", - "email": "ochevalier4@leroux.eu", - "country": "France", - "region": "Provence-Alpes-Côte d'Azur", - "employeeNumber": "25239", - "security": "confidential" - }, - { - "firstName": "Adonise", - "lastName": "GUERIN", - "phone": "05 59 17 83 77", - "email": "celestinesanchez@garnier.net", - "country": "France", - "region": "Corse", - "employeeNumber": "28055-9019", - "security": "confidential" - }, - { - "firstName": "Conception", - "lastName": "REY", - "phone": "00 38 13 17 19", - "email": "jrobert@denis.name", - "country": "France", - "region": "Normandie", - "employeeNumber": "00502", - "security": "confidential" - }, - { - "firstName": "Angadrême", - "lastName": "VINCENT", - "phone": "03 79 86 01 72", - "email": "marceaurichard@durand.eu", - "country": "France", - "region": "Corse", - "employeeNumber": "24186-1062", - "security": "confidential" - }, - { - "firstName": "Delphine", - "lastName": "LE ROUX", - "phone": "04 46 70 39 89", - "email": "wmathieu@renault.fr", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "47263-2382", - "security": "confidential" - }, - { - "firstName": "Gabin", - "lastName": "BOURGEOIS", - "phone": "01 25 52 09 20", - "email": "wcarre22@mathieu.fr", - "country": "France", - "region": "Centre-Val de Loire", - "employeeNumber": "97467-1718", - "security": "confidential" - }, - { - "firstName": "Bérangère", - "lastName": "SCHMITT", - "phone": "01 78 44 09 27", - "email": "ndumont58@clement.eu", - "country": "France", - "region": "Corse", - "employeeNumber": "14655", - "security": "confidential" - }, - { - "firstName": "Arnould", - "lastName": "AUBRY", - "phone": "00 55 12 43 67", - "email": "qduval7@vasseur.info", - "country": "France", - "region": "Provence-Alpes-Côte d'Azur", - "employeeNumber": "95328", - "security": "confidential" - }, - { - "firstName": "Daniel", - "lastName": "VASSEUR", - "phone": "07 60 59 40 06", - "email": "fpierre@andre.com", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "28404-8895", - "security": "confidential" - }, - { - "firstName": "Paulette", - "lastName": "CHEVALIER", - "phone": "05 76 47 98 16", - "email": "deboraguillot@dupuy.net", - "country": "France", - "region": "Auvergne-Rhône-Alpes", - "employeeNumber": "82414-9145", - "security": "confidential" - }, - { - "firstName": "Sidoine", - "lastName": "CARPENTIER", - "phone": "06 13 52 26 40", - "email": "qrenard90@fournier.com", - "country": "France", - "region": "Bretagne", - "employeeNumber": "14011", - "security": "confidential" - }, - { - "firstName": "Béranger", - "lastName": "RODRIGUEZ", - "phone": "07 09 43 00 75", - "email": "taurinnicolas@gerard.org", - "country": "France", - "region": "Grand Est", - "employeeNumber": "61782-1334", - "security": "confidential" - }, - { - "firstName": "Jocelyn", - "lastName": "CARON", - "phone": "02 30 27 26 30", - "email": "kriviere@nicolas.org", - "country": "France", - "region": "Provence-Alpes-Côte d'Azur", - "employeeNumber": "72175", - "security": "confidential" - }, - { - "firstName": "Capucine", - "lastName": "HUBERT", - "phone": "07 24 82 39 79", - "email": "pvasseur2@breton.info", - "country": "France", - "region": "Occitanie", - "employeeNumber": "95394-9443", - "security": "confidential" - }, - { - "firstName": "Aaron", - "lastName": "LECOMTE", - "phone": "09 00 41 25 99", - "email": "trolland9@moreau.eu", - "country": "France", - "region": "Bourgogne-Franche-Comté", - "employeeNumber": "30036", - "security": "confidential" - }, - { - "firstName": "Lazare", - "lastName": "GARCIA", - "phone": "08 42 38 09 23", - "email": "ofontaine9@leclerc.org", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "56617", - "security": "confidential" - }, - { - "firstName": "Alcidie", - "lastName": "CARON", - "phone": "09 83 12 87 61", - "email": "qbonnet8@renard.info", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "94543-9283", - "security": "confidential" - }, - { - "firstName": "Honorine", - "lastName": "COLLET", - "phone": "08 70 45 41 78", - "email": "qbourgeois77@barbier.org", - "country": "France", - "region": "Bourgogne-Franche-Comté", - "employeeNumber": "59761-5344", - "security": "confidential" - }, - { - "firstName": "Joséphine", - "lastName": "LECOMTE", - "phone": "06 83 15 68 98", - "email": "eliafernandez@david.fr", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "06656-3067", - "security": "confidential" - }, - { - "firstName": "Brigitte", - "lastName": "BLANCHARD", - "phone": "01 13 21 70 78", - "email": "zmorel2@noel.com", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "92033-1976", - "security": "confidential" - }, - { - "firstName": "Rodolphe", - "lastName": "BRUN", - "phone": "05 86 79 82 12", - "email": "mpaul83@collet.net", - "country": "France", - "region": "Grand Est", - "employeeNumber": "76431-4612", - "security": "confidential" - }, - { - "firstName": "Clémentine", - "lastName": "VASSEUR", - "phone": "07 70 37 10 76", - "email": "uschneider@schneider.fr", - "country": "France", - "region": "Hauts-de-France", - "employeeNumber": "19780", - "security": "confidential" - }, - { - "firstName": "Amiel", - "lastName": "BRUN", - "phone": "09 45 85 13 17", - "email": "sguillot@bonnet.fr", - "country": "France", - "region": "Nouvelle-Aquitaine", - "employeeNumber": "39043-3916", - "security": "confidential" - }, - { - "firstName": "Macaire", - "lastName": "MARCHAL", - "phone": "03 77 55 25 49", - "email": "artemisaubry@prevost.eu", - "country": "France", - "region": "Bourgogne-Franche-Comté", - "employeeNumber": "44275", - "security": "confidential" - }, - { - "firstName": "Calixte", - "lastName": "MOREL", - "phone": "08 97 35 42 71", - "email": "uprevost14@caron.info", - "country": "France", - "region": "Île-de-France", - "employeeNumber": "72517-4062", - "security": "confidential" - }, - { - "firstName": "Jérôme", - "lastName": "GERARD", - "phone": "00 82 99 69 92", - "email": "brodriguez@cousin.info", - "country": "France", - "region": "Nouvelle-Aquitaine", - "employeeNumber": "02686-6469", - "security": "confidential" - }, - { - "firstName": "Théodora", - "lastName": "BONNET", - "phone": "03 93 02 43 95", - "email": "obourgeois@dufour.com", - "country": "France", - "region": "Provence-Alpes-Côte d'Azur", - "employeeNumber": "88665-7198", - "security": "confidential" - } -] \ No newline at end of file diff --git a/src/upsert.rs b/src/upsert.rs index b826f94e..f3229b72 100644 --- a/src/upsert.rs +++ b/src/upsert.rs @@ -40,7 +40,7 @@ pub trait FindexUpsert< master_key: &KeyingMaterial, label: &Label, items: HashMap>, - ) -> Result, Error> { + ) -> Result, Error> { self.upsert(master_key, label, items, HashMap::new()).await } @@ -59,7 +59,7 @@ pub trait FindexUpsert< master_key: &KeyingMaterial, label: &Label, items: HashMap>, - ) -> Result, Error> { + ) -> Result, Error> { self.upsert(master_key, label, HashMap::new(), items).await } @@ -81,22 +81,19 @@ pub trait FindexUpsert< label: &Label, additions: HashMap>, deletions: HashMap>, - ) -> Result, Error> { + ) -> Result, Error> { check_parameter_constraints::(); let mut rng = CsRng::from_entropy(); let k_uid = master_key.derive_kmac_key::(ENTRY_TABLE_KEY_DERIVATION_INFO); let k_value = master_key.derive_dem_key(ENTRY_TABLE_KEY_DERIVATION_INFO); - // a reverse map of Keyword UIDs (e.g. hashes) to Keywords - let mut uid_to_keyword = HashMap::, Keyword>::new(); - - let mut new_chains = HashMap::>::with_capacity( + let mut modifications = HashMap::>::with_capacity( additions.len() + deletions.len(), ); for (indexed_value, keywords) in additions { for keyword in keywords { - new_chains + modifications .entry(keyword) .or_default() .insert(indexed_value.clone(), BlockType::Addition); @@ -104,26 +101,25 @@ pub trait FindexUpsert< } for (indexed_value, keywords) in deletions { for keyword in keywords { - new_chains + modifications .entry(keyword) .or_default() .insert(indexed_value.clone(), BlockType::Deletion); } } - // Compute the Entry Table UIDs. - let mut new_chains = new_chains - .into_iter() - .map(|(keyword, indexed_values)| { - let keyword_hash = keyword.hash(); - let uid = EntryTable::::generate_uid::( - &k_uid, - &keyword_hash, - label, - ); - uid_to_keyword.insert(uid, keyword); - (uid, (keyword_hash, indexed_values)) - }) - .collect::>(); + + let mut uid_to_keyword = HashMap::with_capacity(modifications.len()); + let mut new_chains = HashMap::with_capacity(modifications.len()); + for (keyword, indexed_values) in modifications { + let keyword_hash = keyword.hash(); + let uid = EntryTable::::generate_uid::( + &k_uid, + &keyword_hash, + label, + ); + uid_to_keyword.insert(uid, keyword); + new_chains.insert(uid, (keyword_hash, indexed_values)); + } // Query the Entry Table for these UIDs. let mut encrypted_entry_table: EncryptedTable = self @@ -131,19 +127,11 @@ pub trait FindexUpsert< .await? .try_into()?; - // compute the map of keywords to booleans indicating whether the keyword - // was already present in the database - let mut keyword_presence = HashMap::new(); - for uid in encrypted_entry_table.keys() { - if let Some(keyword) = uid_to_keyword.remove(uid) { - keyword_presence.insert(keyword, true); - } - } - // whatever is left in `uid_to_keyword` is not in the database, - // i.e. not already present. Update the keyword_presence accordingly - for (_, keyword) in uid_to_keyword { - keyword_presence.insert(keyword, false); - } + let new_keywords = uid_to_keyword + .into_iter() + .filter(|(uid, _)| !encrypted_entry_table.contains_key(uid)) + .map(|(_, keyword)| keyword) + .collect(); while !new_chains.is_empty() { // Decrypt the Entry Table once and for all. @@ -171,7 +159,7 @@ pub trait FindexUpsert< new_chains.retain(|uid, _| encrypted_entry_table.contains_key(uid)); } - Ok(keyword_presence) + Ok(new_keywords) } /// Writes the given modifications to the indexes. Returns the current value diff --git a/tests/test_in_memory.rs b/tests/test_in_memory.rs index d5cbb633..b4095256 100644 --- a/tests/test_in_memory.rs +++ b/tests/test_in_memory.rs @@ -683,100 +683,6 @@ async fn test_graph_compacting() { } } -#[cfg(feature = "live_compact")] -#[actix_rt::test] -async fn test_live_compacting() { - use cosmian_findex::FindexLiveCompact; - - let mut rng = CsRng::from_entropy(); - let mut findex = FindexInMemory::default(); - - let label = Label::random(&mut rng); - let master_key = KeyingMaterial::new(&mut rng); - - let doe_keyword = Keyword::from(b"doe".to_vec()); - let robert_keyword = Keyword::from(b"robert".to_vec()); - let robert_doe_location = Location::from("robert doe DB location"); - - // Direct location robert doe. - let mut indexed_value_to_keywords = HashMap::new(); - indexed_value_to_keywords.insert( - IndexedValue::Location(robert_doe_location.clone()), - HashSet::from_iter([robert_keyword.clone(), doe_keyword.clone()]), - ); - - for _ in 0..100 { - // Add some keywords. - findex - .add(&master_key, &label, indexed_value_to_keywords.clone()) - .await - .unwrap(); - - // Remove them. - findex - .remove(&master_key, &label, indexed_value_to_keywords.clone()) - .await - .unwrap(); - } - - // Add some keywords. - findex - .add(&master_key, &label, indexed_value_to_keywords.clone()) - .await - .unwrap(); - - // Search Robert. - let robert_search = findex - .search( - &master_key, - &label, - HashSet::from_iter([robert_keyword.clone()]), - ) - .await - .unwrap(); - - check_search_result(&robert_search, &robert_keyword, &robert_doe_location); - - // Search Doe. - let doe_search = findex - .search( - &master_key, - &label, - HashSet::from_iter([doe_keyword.clone()]), - ) - .await - .unwrap(); - check_search_result(&doe_search, &doe_keyword, &robert_doe_location); - - // Compact enough times to be sure all entries have been compacted. - for _ in 0..10 { - findex.live_compact(&master_key, 8).await.unwrap(); - } - - // After compaction, there should still be two entries in the Entry Table. - assert_eq!(findex.entry_table_len(), 2); - // But deletions should have been simplified (only two locations indexed per - // chain -> one line per chain -> 2 lines) - assert_eq!(findex.chain_table_len(), 2); - - for _ in 0..100 { - // Add some keywords. - findex - .add(&master_key, &label, indexed_value_to_keywords.clone()) - .await - .unwrap(); - } - assert_eq!(findex.entry_table_len(), 2); - assert_eq!(findex.chain_table_len(), 202); - - // Compact enough times to be sure all entries have been compacted. - for _ in 0..10 { - findex.live_compact(&master_key, 8).await.unwrap(); - } - assert_eq!(findex.entry_table_len(), 2); - assert_eq!(findex.chain_table_len(), 2); -} - #[actix_rt::test] async fn test_search_cyclic_graph() { let mut rng = CsRng::from_entropy(); @@ -926,9 +832,7 @@ async fn test_search_cyclic_graph() { #[actix_rt::test] async fn test_keyword_presence() -> Result<(), Error> { let mut rng = CsRng::from_entropy(); - let label = Label::random(&mut rng); - let master_key = KeyingMaterial::new(&mut rng); let mut indexed_value_to_keywords = HashMap::new(); @@ -948,17 +852,15 @@ async fn test_keyword_presence() -> Result<(), Error> { ); let findex = FindexInMemory::default(); - let presence = findex + let new_keywords = findex .add(&master_key, &label, indexed_value_to_keywords) .await?; + // the 3 keywords should not be present in the database - assert_eq!(presence.len(), 3); - assert!(presence.contains_key(&Keyword::from("robert"))); - assert!(presence.contains_key(&Keyword::from("doe"))); - assert!(presence.contains_key(&Keyword::from("john"))); - assert!(!*presence.get(&Keyword::from("robert")).unwrap()); - assert!(!*presence.get(&Keyword::from("doe")).unwrap()); - assert!(!*presence.get(&Keyword::from("john")).unwrap()); + assert_eq!(new_keywords.len(), 3); + assert!(new_keywords.contains(&Keyword::from("robert"))); + assert!(new_keywords.contains(&Keyword::from("doe"))); + assert!(new_keywords.contains(&Keyword::from("john"))); // Now insert a Robert Smith let mut indexed_value_to_keywords = HashMap::new(); @@ -967,48 +869,37 @@ async fn test_keyword_presence() -> Result<(), Error> { IndexedValue::Location(robert_smith_location), hashset_keywords(&["robert", "smith"]), ); - let presence = findex + let new_keywords = findex .add(&master_key, &label, indexed_value_to_keywords) .await?; // robert should be present, but not smith - assert_eq!(presence.len(), 2); - assert!(presence.contains_key(&Keyword::from("robert"))); - assert!(presence.contains_key(&Keyword::from("smith"))); - assert!(*presence.get(&Keyword::from("robert")).unwrap()); - assert!(!*presence.get(&Keyword::from("smith")).unwrap()); + assert_eq!(new_keywords.len(), 1); + assert!(!new_keywords.contains(&Keyword::from("robert"))); + assert!(new_keywords.contains(&Keyword::from("smith"))); - // Delete Robert Smith but add the Junior keyword + // Delete Robert Smith and the junior keyword let robert_smith_location = Location::from("robert smith DB location"); let mut indexed_value_to_keywords = HashMap::new(); indexed_value_to_keywords.insert( IndexedValue::Location(robert_smith_location.clone()), hashset_keywords(&["robert", "smith", "junior"]), ); - let presence = findex + let new_keywords = findex .remove(&master_key, &label, indexed_value_to_keywords.clone()) .await?; // robert and smith should be present, but not junior - assert_eq!(presence.len(), 3); - assert!(presence.contains_key(&Keyword::from("robert"))); - assert!(presence.contains_key(&Keyword::from("smith"))); - assert!(presence.contains_key(&Keyword::from("junior"))); - assert!(*presence.get(&Keyword::from("robert")).unwrap()); - assert!(*presence.get(&Keyword::from("smith")).unwrap()); - assert!(!*presence.get(&Keyword::from("junior")).unwrap()); + assert_eq!(new_keywords.len(), 1); + assert!(!new_keywords.contains(&Keyword::from("robert"))); + assert!(!new_keywords.contains(&Keyword::from("smith"))); + assert!(new_keywords.contains(&Keyword::from("junior"))); // however, the first delete create an entry for "junior, // therefore deleting again will find it - let presence = findex + let new_keywords = findex .remove(&master_key, &label, indexed_value_to_keywords.clone()) .await?; - // robert and smith should be present, but not junior - assert_eq!(presence.len(), 3); - assert!(presence.contains_key(&Keyword::from("robert"))); - assert!(presence.contains_key(&Keyword::from("smith"))); - assert!(presence.contains_key(&Keyword::from("junior"))); - assert!(*presence.get(&Keyword::from("robert")).unwrap()); - assert!(*presence.get(&Keyword::from("smith")).unwrap()); - assert!(*presence.get(&Keyword::from("junior")).unwrap()); + // all should be present + assert_eq!(new_keywords.len(), 0); Ok(()) } From c85c81afdb91347892c5ea43baa9c7995bc6790e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Tue, 1 Aug 2023 10:32:03 +0200 Subject: [PATCH 8/9] fix: upsert functions' documentation --- src/upsert.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/upsert.rs b/src/upsert.rs index f3229b72..f43b91c2 100644 --- a/src/upsert.rs +++ b/src/upsert.rs @@ -27,14 +27,13 @@ pub trait FindexUpsert< { /// Index the given values for the associated keywords. /// + /// Returns the set of the new keywords. + /// /// # Parameters /// /// - `master_key` : Findex master key /// - `label` : additional public information used in key hashing /// - `items` : set of keywords used to index values - /// - /// Returns a map of keywords to booleans indicating whether the keyword - /// was already present in the database. async fn add( &self, master_key: &KeyingMaterial, @@ -46,14 +45,13 @@ pub trait FindexUpsert< /// Removes the given values from the indexes for the associated keywords. /// + /// Returns the set of the new keywords. + /// /// # Parameters /// /// - `master_key` : Findex master key /// - `label` : additional public information used in key hashing /// - `items` : set of keywords used to index values - /// - /// Returns a map of keywords to booleans indicating whether the keyword - /// was already present in the database. async fn remove( &self, master_key: &KeyingMaterial, @@ -65,6 +63,8 @@ pub trait FindexUpsert< /// Upsert the given chain elements in Findex tables. /// + /// Returns the set of the new keywords. + /// /// # Parameters /// /// - `master_key` : Findex master key @@ -72,9 +72,6 @@ pub trait FindexUpsert< /// - `additions` : values to indexed for a set of keywords /// - `deletions` : values to remove from the indexes for a set of /// keywords - /// - /// Returns a map of keywords to booleans indicating whether the keyword - /// was already present in the database. async fn upsert( &self, master_key: &KeyingMaterial, From 367439b213aaf0cf32c7a13fa56458639fa7ef1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Tue, 1 Aug 2023 11:13:54 +0200 Subject: [PATCH 9/9] fix: rebase on develop --- src/in_memory_example.rs | 10 +++++----- tests/non_regression.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/in_memory_example.rs b/src/in_memory_example.rs index 8098f521..06c38e64 100644 --- a/src/in_memory_example.rs +++ b/src/in_memory_example.rs @@ -96,19 +96,19 @@ impl FindexInMemory { pub fn dump_tables(&self) -> Result, ExampleError> { let mut ser = Serializer::new(); - ser.write(&self.entry_table) + ser.write(&*self.entry_table.read().expect("entry table lock poisoned")) .map_err(|e| ExampleError(e.to_string()))?; - ser.write(&self.chain_table) + ser.write(&*self.chain_table.read().expect("chain table lock poisoned")) .map_err(|e| ExampleError(e.to_string()))?; Ok(ser.finalize().to_vec()) } - pub fn load_tables(&mut self, bytes: &[u8]) -> Result<(), ExampleError> { + pub fn load_tables(&self, bytes: &[u8]) -> Result<(), ExampleError> { let mut de = Deserializer::new(bytes); - self.entry_table = de + *self.entry_table.write().expect("entry table lock poisoned") = de .read::>() .map_err(|e| ExampleError(e.to_string()))?; - self.chain_table = de + *self.chain_table.write().expect("chain table lock poisoned") = de .read::>() .map_err(|e| ExampleError(e.to_string()))?; if !de.finalize().is_empty() { diff --git a/tests/non_regression.rs b/tests/non_regression.rs index 813d3676..c2e59b82 100644 --- a/tests/non_regression.rs +++ b/tests/non_regression.rs @@ -41,7 +41,7 @@ async fn write_index() -> Result<(), Error> { let master_key = KeyingMaterial::new(&mut rng); let label = Label::random(&mut rng); - let mut findex = FindexInMemory::default(); + let findex = FindexInMemory::default(); let reader = BufReader::new(File::open("datasets/first_names.txt").unwrap()); for maybe_line in reader.lines().take(MAX_FIRST_NAMES) { @@ -87,7 +87,7 @@ async fn test_non_regression() -> Result<(), Error> { // Uncomment to generate new test data. // write_index().await?; - let mut findex = FindexInMemory::::default(); + let findex = FindexInMemory::::default(); let serialized_index = std::fs::read("datasets/serialized_index").unwrap(); findex.load_tables(&serialized_index)?;