#522 Return iterator from all_resources

lib/src/db.rs

@@ -13,7 +13,7 @@ use crate::{
     endpoints::{default_endpoints, Endpoint},
     errors::{AtomicError, AtomicResult},
     resources::PropVals,
-    storelike::{Query, QueryResult, ResourceCollection, Storelike},
+    storelike::{Query, QueryResult, Storelike},
     Atom, Resource, Value,
 };
 
@@ -164,6 +164,24 @@ impl Db {
         self.watched_queries.clear()?;
         Ok(())
     }
+
+    fn map_sled_item_to_resource(
+        item: Result<(sled::IVec, sled::IVec), sled::Error>,
+        self_url: String,
+        include_external: bool,
+    ) -> Option<Resource> {
+        let (subject, resource_bin) = item.expect(DB_CORRUPT_MSG);
+        let subject: String = String::from_utf8_lossy(&subject).to_string();
+
+        if !include_external && !subject.starts_with(&self_url) {
+            return None;
+        }
+
+        let propvals: PropVals = bincode::deserialize(&resource_bin)
+            .unwrap_or_else(|e| panic!("{}. {}", corrupt_db_message(&subject), e));
+
+        Some(Resource::from_propvals(propvals, subject))
+    }
 }
 
 impl Storelike for Db {
@@ -514,23 +532,19 @@ impl Storelike for Db {
     }
 
     #[instrument(skip(self))]
-    fn all_resources(&self, include_external: bool) -> ResourceCollection {
-        let mut resources: ResourceCollection = Vec::new();
+    fn all_resources(
+        &self,
+        include_external: bool,
+    ) -> Box<dyn std::iter::Iterator<Item = Resource>> {
         let self_url = self
             .get_self_url()
             .expect("No self URL set, is required in DB");
-        for item in self.resources.into_iter() {
-            let (subject, resource_bin) = item.expect(DB_CORRUPT_MSG);
-            let subject: String = String::from_utf8_lossy(&subject).to_string();
-            if !include_external && !subject.starts_with(&self_url) {
-                continue;
-            }
-            let propvals: PropVals = bincode::deserialize(&resource_bin)
-                .unwrap_or_else(|e| panic!("{}. {}", corrupt_db_message(&subject), e));
-            let resource = Resource::from_propvals(propvals, subject);
-            resources.push(resource);
-        }
-        resources
+
+        let result = self.resources.into_iter().filter_map(move |item| {
+            Db::map_sled_item_to_resource(item, self_url.clone(), include_external)
+        });
+
+        Box::new(result)
     }
 
     fn populate(&self) -> AtomicResult<()> {

lib/src/db/test.rs

@@ -52,7 +52,9 @@ fn basic() {
     // Should throw an error, because resource is deleted
     store.get_propvals(crate::urls::CLASS).unwrap_err();
 
-    assert!(store.all_resources(false).len() < store.all_resources(true).len());
+    let all_local_resources = store.all_resources(false).count();
+    let all_resources = store.all_resources(true).count();
+    assert!(all_local_resources < all_resources);
 }
 
 #[test]
@@ -429,8 +431,8 @@ fn query_include_external() {
 #[test]
 fn test_db_resources_all() {
     let store = &Db::init_temp("resources_all").unwrap();
-    let res_no_include = store.all_resources(false).len();
-    let res_include = store.all_resources(true).len();
+    let res_no_include = store.all_resources(false).count();
+    let res_include = store.all_resources(true).count();
     assert!(
         res_include > res_no_include,
         "Amount of results should be higher for include_external"

lib/src/parse.rs

@@ -427,13 +427,14 @@ mod test {
         use crate::Storelike;
         let store1 = crate::Store::init().unwrap();
         store1.populate().unwrap();
-        let serialized =
-            crate::serialize::resources_to_json_ad(&store1.all_resources(true)).unwrap();
         let store2 = crate::Store::init().unwrap();
+        let all1: Vec<Resource> = store1.all_resources(true).collect();
+        let serialized = crate::serialize::resources_to_json_ad(&all1).unwrap();
+
         store2.import(&serialized, &ParseOpts::default()).unwrap();
-        let all1 = store1.all_resources(true);
-        let all2 = store2.all_resources(true);
-        assert_eq!(all1.len(), all2.len());
+        let all2_count = store2.all_resources(true).count();
+
+        assert_eq!(all1.len(), all2_count);
         let found_shortname = store2
             .get_resource(urls::CLASS)
             .unwrap()

lib/src/store.rs

@@ -1,10 +1,7 @@
 //! In-memory store of Atomic data.
 //! This provides many methods for finding, changing, serializing and parsing Atomic Data.
 
-use crate::{
-    atoms::Atom,
-    storelike::{ResourceCollection, Storelike},
-};
+use crate::{atoms::Atom, storelike::Storelike};
 use crate::{errors::AtomicResult, Resource};
 use std::{collections::HashMap, sync::Arc, sync::Mutex};
 
@@ -79,12 +76,15 @@ impl Storelike for Store {
     }
 
     // TODO: Fix this for local stores, include external does not make sense here
-    fn all_resources(&self, _include_external: bool) -> ResourceCollection {
-        let mut all = Vec::new();
-        for (_subject, resource) in self.hashmap.lock().unwrap().clone().into_iter() {
-            all.push(resource)
-        }
-        all
+    fn all_resources(&self, _include_external: bool) -> Box<dyn Iterator<Item = Resource>> {
+        Box::new(
+            self.hashmap
+                .lock()
+                .unwrap()
+                .clone()
+                .into_iter()
+                .map(|(_subject, resource)| resource),
+        )
     }
 
     fn get_server_url(&self) -> &str {

lib/src/storelike.rs

@@ -59,10 +59,9 @@ pub trait Storelike: Sized {
         overwrite_existing: bool,
     ) -> AtomicResult<()>;
 
-    /// Returns a collection with all resources in the store.
+    /// Returns an iterator that iterates over all resources in the store.
     /// If Include_external is false, this is filtered by selecting only resoureces that match the `self` URL of the store.
-    /// WARNING: This could be very expensive!
-    fn all_resources(&self, include_external: bool) -> ResourceCollection;
+    fn all_resources(&self, include_external: bool) -> Box<dyn Iterator<Item = Resource>>;
 
     /// Constructs the value index from all resources in the store. Could take a while.
     fn build_index(&self, include_external: bool) -> AtomicResult<()> {

server/src/search.rs

@@ -115,14 +115,14 @@ pub fn get_schema_fields(appstate: &SearchState) -> AtomicServerResult<Fields> {
 /// Indexes all resources from the store to search.
 /// At this moment does not remove existing index.
 pub fn add_all_resources(search_state: &SearchState, store: &Db) -> AtomicServerResult<()> {
-    for resource in store.all_resources(true) {
-        // Skip commits
-        // TODO: Better check, this might overfit
-        if resource.get_subject().contains("/commits/") {
-            continue;
-        }
+    let resources = store
+        .all_resources(true)
+        .filter(|resource| !resource.get_subject().contains("/commits/"));
+
+    for resource in resources {
         add_resource(search_state, &resource, store)?;
     }
+
     search_state.writer.write()?.commit()?;
     Ok(())
 }