From ef9f32202ca4959982aa1f0438a950359ded3a31 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Sun, 18 Jun 2023 10:01:55 -0400 Subject: [PATCH 01/32] combine query-filter xf and flake xf for efficiency --- src/fluree/db/query/range.cljc | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/fluree/db/query/range.cljc b/src/fluree/db/query/range.cljc index f0f9cbe7b..b3b3af6ef 100644 --- a/src/fluree/db/query/range.cljc +++ b/src/fluree/db/query/range.cljc @@ -126,18 +126,17 @@ flakes from the leaf nodes in the input stream, with one flake collection for each input leaf." [{:keys [start-flake start-test end-flake end-test flake-xf] :as opts}] - (let [query-xf (comp (map :flakes) - (map (fn [flakes] - (flake/subrange flakes - start-test start-flake - end-test end-flake))) - (map (fn [flakes] - (into [] (query-filter opts) flakes))))] - (if flake-xf - (let [slice-xf (map (fn [flakes] - (sequence flake-xf flakes)))] - (comp query-xf slice-xf)) - query-xf))) + (let [flake-xfs (cond-> [(query-filter opts)] + flake-xf (conj flake-xf)) + flake-xf* (apply comp flake-xfs) + query-xf (comp (map :flakes) + (map (fn [flakes] + (flake/subrange flakes + start-test start-flake + end-test end-flake))) + (map (fn [flakes] + (into [] flake-xf* flakes))))] + query-xf)) (defn resolve-flake-slices "Returns a channel that will contain a stream of chunked flake collections that From f7622ce7a47148b5b0a6eeb007f627e2b4443d14 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Sun, 18 Jun 2023 21:01:50 -0400 Subject: [PATCH 02/32] remove deprecated :block parameter --- src/fluree/db/index.cljc | 2 -- src/fluree/db/indexer/default.cljc | 1 - src/fluree/db/json_ld/transact.cljc | 1 - src/fluree/db/query/sql.cljc | 2 +- src/fluree/db/storage/core.cljc | 6 +++--- src/fluree/db/util/pprint.clj | 2 +- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index 88bc13254..60408da4a 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -121,7 +121,6 @@ :first flake/maximum :rhs nil :size 0 - :block 0 :t 0 :leftmost? true}) @@ -175,7 +174,6 @@ :rhs nil :children children :size 0 - :block 0 :t 0 :leftmost? true})) diff --git a/src/fluree/db/indexer/default.cljc b/src/fluree/db/indexer/default.cljc index 3834eba0b..ca6d8da4a 100644 --- a/src/fluree/db/indexer/default.cljc +++ b/src/fluree/db/indexer/default.cljc @@ -460,7 +460,6 @@ init-stats {:network network :ledger-id ledger-id :t t - :block block :novelty-size novelty-size :start-time (util/current-time-iso)}] (if (or (dirty? db) diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index 8946c11e7..c9f45f772 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -250,7 +250,6 @@ :stage-update? (= t db-t) ;; if a previously staged db is getting updated again before committed :refs (volatile! (or (:refs schema) #{const/$rdf:type})) :t t - :block block :last-pid last-pid :last-sid last-sid :next-pid (fn [] (vswap! last-pid inc)) diff --git a/src/fluree/db/query/sql.cljc b/src/fluree/db/query/sql.cljc index 58e147074..cd1676aca 100644 --- a/src/fluree/db/query/sql.cljc +++ b/src/fluree/db/query/sql.cljc @@ -491,5 +491,5 @@ sql parse-rule first - (select-keys [:select :selectDistinct :selectOne :where :block :prefixes + (select-keys [:select :selectDistinct :selectOne :where :prefixes :vars :opts]))) diff --git a/src/fluree/db/storage/core.cljc b/src/fluree/db/storage/core.cljc index 077b95f6b..f6f287eb4 100644 --- a/src/fluree/db/storage/core.cljc +++ b/src/fluree/db/storage/core.cljc @@ -78,7 +78,7 @@ (let [{:keys [conn ledger ledger-id t]} db t' (- t) ;; use positive t integer data {:ledger-id ledger-id - :block t' + :t t' :garbage garbage} ser (serdeproto/-serialize-garbage (serde conn) data)] ( branch-metadata (assoc :leftmost? (and leftmost? diff --git a/src/fluree/db/util/pprint.clj b/src/fluree/db/util/pprint.clj index 86d15db47..dba1c367f 100644 --- a/src/fluree/db/util/pprint.clj +++ b/src/fluree/db/util/pprint.clj @@ -17,7 +17,7 @@ str-vec (if (index/leaf? index) (let [node-count (count (:flakes index))] (swap! count-atom + node-count) - (conj str-vec (str ":" (:block index) "-" node-count))) + (conj str-vec (str ":" (:t index) "-" node-count))) str-vec) first-flake (:first index) main-str (apply str str-vec) From 02ee48d938b41dfd44ac72465ade972bf29a6518 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Thu, 22 Jun 2023 10:14:34 -0400 Subject: [PATCH 03/32] remove unused fns --- dev/json_ld/shacl.clj | 1 - src/fluree/db/index.cljc | 21 --------------------- 2 files changed, 22 deletions(-) diff --git a/dev/json_ld/shacl.clj b/dev/json_ld/shacl.clj index 397a3b15e..7a058b59d 100644 --- a/dev/json_ld/shacl.clj +++ b/dev/json_ld/shacl.clj @@ -18,7 +18,6 @@ [fluree.db.util.log :as log] [fluree.db.index :as index] [criterium.core :as criterium] - [clojure.data.avl :as avl] [clojure.tools.reader.edn :as edn])) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index 60408da4a..419c13d8b 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -63,27 +63,6 @@ (leaf? node) (dissoc node :flakes) (branch? node) (dissoc node :children))) -(defn lookup - [branch flake] - (when (and (branch? branch) - (resolved? branch)) - (let [{:keys [children]} branch] - (-> children - (avl/nearest <= flake) - (or (first children)) - val)))) - -(defn lookup-leaf - [r branch flake] - (go-try - (when (and (branch? branch) - (resolved? branch)) - (loop [child (lookup branch flake)] - (if (leaf? child) - child - (recur ( leaf From 16a066ef551270e76a96a63ce63a74a98afcc725 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Thu, 22 Jun 2023 10:16:38 -0400 Subject: [PATCH 04/32] use subrange fn defined in flake ns --- src/fluree/db/index.cljc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index 419c13d8b..bc2da0dde 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -1,7 +1,6 @@ (ns fluree.db.index (:refer-clojure :exclude [resolve]) - (:require [clojure.data.avl :as avl] - [fluree.db.flake :as flake] + (:require [fluree.db.flake :as flake] #?(:clj [clojure.core.async :refer [chan go !] :as async] :cljs [cljs.core.async :refer [chan go !] :as async]) [fluree.db.util.async :refer [ first-flake <= rhs) + (flake/subrange novelty > first-flake <= rhs) ;; right only boundary (and rhs leftmost?) - (avl/subrange novelty <= rhs) + (flake/subrange novelty <= rhs) ;; left only boundary (and (nil? rhs) (not leftmost?)) - (avl/subrange novelty > first-flake) + (flake/subrange novelty > first-flake) ;; no boundary (and (nil? rhs) leftmost?) From b1971b94a40a986fae6c2ed27cd406946e82dde5 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Thu, 22 Jun 2023 10:24:02 -0400 Subject: [PATCH 05/32] remove unused fns and references to network attribute --- src/fluree/db/db/json_ld.cljc | 10 ++++----- src/fluree/db/index.cljc | 38 +++++++---------------------------- 2 files changed, 12 insertions(+), 36 deletions(-) diff --git a/src/fluree/db/db/json_ld.cljc b/src/fluree/db/db/json_ld.cljc index 6827b9c74..ef1410601 100644 --- a/src/fluree/db/db/json_ld.cljc +++ b/src/fluree/db/db/json_ld.cljc @@ -313,11 +313,11 @@ opst-cmp :opst tspo-cmp :tspo} index/default-comparators - spot (index/empty-branch method alias spot-cmp) - psot (index/empty-branch method alias psot-cmp) - post (index/empty-branch method alias post-cmp) - opst (index/empty-branch method alias opst-cmp) - tspo (index/empty-branch method alias tspo-cmp) + spot (index/empty-branch alias spot-cmp) + psot (index/empty-branch alias psot-cmp) + post (index/empty-branch alias post-cmp) + opst (index/empty-branch alias opst-cmp) + tspo (index/empty-branch alias tspo-cmp) stats {:flakes 0, :size 0, :indexed 0} schema (vocab/base-schema) branch (branch/branch-meta ledger) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index bc2da0dde..c504820f2 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -89,10 +89,9 @@ (defn empty-leaf "Returns a blank leaf node map for the provided `network`, `ledger-id`, and index comparator `cmp`." - [network ledger-id cmp] + [ledger-alias cmp] {:comparator cmp - :network network - :ledger-id ledger-id + :ledger-id ledger-alias :id :empty :tempid (random-uuid) :leaf true @@ -102,18 +101,11 @@ :t 0 :leftmost? true}) -(defn new-leaf - [network ledger-id cmp flakes] - (let [empty-set (flake/sorted-set-by cmp)] - (-> (empty-leaf network ledger-id cmp) - (assoc :flakes empty-set) - (add-flakes flakes)))) - (defn descendant? "Checks if the `node` passed in the second argument is a descendant of the `branch` passed in the first argument" [{:keys [rhs leftmost?], cmp :comparator, first-flake :first, :as branch} - {node-first :first, node-rhs :rhs, :as node}] + {node-first :first, node-rhs :rhs, :as _node}] (if-not (branch? branch) false (and (or leftmost? @@ -139,12 +131,11 @@ (defn empty-branch "Returns a blank branch node which contains a single empty leaf node for the provided `network`, `ledger-id`, and index comparator `cmp`." - [network ledger-id cmp] - (let [child-node (empty-leaf network ledger-id cmp) + [ledger-alias cmp] + (let [child-node (empty-leaf ledger-alias cmp) children (child-map cmp child-node)] {:comparator cmp - :network network - :ledger-id ledger-id + :ledger-id ledger-alias :id :empty :tempid (random-uuid) :leaf false @@ -155,21 +146,6 @@ :t 0 :leftmost? true})) -(defn reset-children - [{:keys [comparator size] :as branch} new-child-nodes] - (let [new-kids (apply child-map comparator new-child-nodes) - new-first (or (some-> new-kids first key) - flake/maximum) - new-size (->> new-child-nodes - (map :size) - (reduce + size))] - (assoc branch :first new-first, :size new-size, :children new-kids))) - -(defn new-branch - [network ledger-id cmp child-nodes] - (-> (empty-branch network ledger-id cmp) - (reset-children child-nodes))) - (defn after-t? "Returns `true` if `flake` has a transaction value after the provided `t`" [t flake] @@ -257,7 +233,7 @@ (defrecord TRangeResolver [node-resolver novelty from-t to-t] Resolver - (resolve [_ {:keys [id tempid tt-id] :as node}] + (resolve [_ node] (if (branch? node) (resolve node-resolver node) (resolve-t-range node-resolver node novelty from-t to-t)))) From d88f4b10677496b894565cb505d2f1abbce5f01a Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Thu, 22 Jun 2023 10:41:41 -0400 Subject: [PATCH 06/32] ledger-id -> ledger-alias --- src/fluree/db/db/json_ld.cljc | 2 +- src/fluree/db/flake.cljc | 1 + src/fluree/db/index.cljc | 48 +++++++++++++++--------------- src/fluree/db/indexer/default.cljc | 9 +++--- src/fluree/db/query/fql.cljc | 4 +-- 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/fluree/db/db/json_ld.cljc b/src/fluree/db/db/json_ld.cljc index ef1410601..60df88b46 100644 --- a/src/fluree/db/db/json_ld.cljc +++ b/src/fluree/db/db/json_ld.cljc @@ -327,7 +327,7 @@ (map->JsonLdDb {:ledger ledger :conn conn :method method - :alias alias + :ledger-alias alias :branch (:name branch) :commit (:commit branch) :t 0 diff --git a/src/fluree/db/flake.cljc b/src/fluree/db/flake.cljc index 7a3f080b1..50d867320 100644 --- a/src/fluree/db/flake.cljc +++ b/src/fluree/db/flake.cljc @@ -1,6 +1,7 @@ (ns fluree.db.flake (:refer-clojure :exclude [split-at sorted-set-by sorted-map-by take last]) (:require [clojure.data.avl :as avl] + [me.tonsky.persistent-sorted-set :as pss] [fluree.db.constants :as const] [fluree.db.util.core :as util] #?(:clj [clojure.pprint :as pprint])) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index c504820f2..85d0b9b8d 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -43,7 +43,7 @@ (catch* e (log/error e "Error resolving index node:" - (select-keys node [:id :network :ledger-id])) + (select-keys node [:id :ledger-aliasd])) (>! error-ch e))))) (defn resolved? @@ -87,19 +87,19 @@ (assoc new-leaf :first new-first))) (defn empty-leaf - "Returns a blank leaf node map for the provided `network`, `ledger-id`, and index + "Returns a blank leaf node map for the provided `ledger-alias` and index comparator `cmp`." [ledger-alias cmp] - {:comparator cmp - :ledger-id ledger-alias - :id :empty - :tempid (random-uuid) - :leaf true - :first flake/maximum - :rhs nil - :size 0 - :t 0 - :leftmost? true}) + {:comparator cmp + :ledger-alias ledger-alias + :id :empty + :tempid (random-uuid) + :leaf true + :first flake/maximum + :rhs nil + :size 0 + :t 0 + :leftmost? true}) (defn descendant? "Checks if the `node` passed in the second argument is a descendant of the @@ -130,21 +130,21 @@ (defn empty-branch "Returns a blank branch node which contains a single empty leaf node for the - provided `network`, `ledger-id`, and index comparator `cmp`." + provided `ledger-alias` and index comparator `cmp`." [ledger-alias cmp] (let [child-node (empty-leaf ledger-alias cmp) children (child-map cmp child-node)] - {:comparator cmp - :ledger-id ledger-alias - :id :empty - :tempid (random-uuid) - :leaf false - :first flake/maximum - :rhs nil - :children children - :size 0 - :t 0 - :leftmost? true})) + {:comparator cmp + :ledger-alias ledger-alias + :id :empty + :tempid (random-uuid) + :leaf false + :first flake/maximum + :rhs nil + :children children + :size 0 + :t 0 + :leftmost? true})) (defn after-t? "Returns `true` if `flake` has a transaction value after the provided `t`" diff --git a/src/fluree/db/indexer/default.cljc b/src/fluree/db/indexer/default.cljc index a565eabae..f94bebdb6 100644 --- a/src/fluree/db/indexer/default.cljc +++ b/src/fluree/db/indexer/default.cljc @@ -213,7 +213,7 @@ (if (overflow-leaf? leaf) (let [target-size (/ *overflow-bytes* 2)] (log/debug "Rebalancing index leaf:" - (select-keys leaf [:id :network :ledger-id])) + (select-keys leaf [:id :ledger-alias])) (loop [[f & r] flakes cur-size 0 cur-first f @@ -345,7 +345,7 @@ "Writes `node` to storage, and puts any errors onto the `error-ch`" [db idx node error-ch updated-ids] (let [node (dissoc node ::old-id) - display-node (select-keys node [:id :ledger-id])] + display-node (select-keys node [:id :ledger-alias])] (async/go (try* (if (index/leaf? node) @@ -451,13 +451,12 @@ (defn refresh [indexer - {:keys [ecount novelty t network ledger-id] :as db} + {:keys [ecount novelty t ledger-alias] :as db} {:keys [remove-preds changes-ch]}] (go-try (let [start-time-ms (util/current-time-millis) novelty-size (:size novelty) - init-stats {:network network - :ledger-id ledger-id + init-stats {:ledger-alias ledger-alias :t t :novelty-size novelty-size :start-time (util/current-time-iso)}] diff --git a/src/fluree/db/query/fql.cljc b/src/fluree/db/query/fql.cljc index 68006dd1e..1eb4a6a70 100644 --- a/src/fluree/db/query/fql.cljc +++ b/src/fluree/db/query/fql.cljc @@ -15,11 +15,11 @@ (defn cache-query "Returns already cached query from cache if available, else executes and stores query into cache." - [{:keys [network ledger-id t auth conn] :as db} query-map] + [{:keys [ledger-alias t auth conn] :as db} query-map] ;; TODO - if a cache value exists, should max-fuel still be checked and throw if not enough? (let [oc (:object-cache conn) query* (update query-map :opts dissoc :fuel :max-fuel) - cache-key [:query network ledger-id t auth query*]] + cache-key [:query ledger-alias t auth query*]] ;; object cache takes (a) key and (b) fn to retrieve value if null (oc cache-key (fn [_] From 7fb543ef9c8174d896f10f7c63d1910a561f8014 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Thu, 22 Jun 2023 12:26:53 -0400 Subject: [PATCH 07/32] remove references to network/ledger-id in storage --- src/fluree/db/serde/json.cljc | 7 +------ src/fluree/db/storage/core.cljc | 35 ++++++++++++++++----------------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/fluree/db/serde/json.cljc b/src/fluree/db/serde/json.cljc index 81614844f..8f8ef23d5 100644 --- a/src/fluree/db/serde/json.cljc +++ b/src/fluree/db/serde/json.cljc @@ -1,9 +1,6 @@ (ns fluree.db.serde.json (:require [fluree.db.serde.protocol :as serdeproto] - [fluree.db.util.json :as json] [fluree.db.flake :as flake] - [fluree.db.util.log :as log :include-macros true] - [clojure.string :as str] [fluree.db.util.core :as util])) #?(:clj (set! *warn-on-reflection* true)) @@ -79,9 +76,7 @@ (defrecord Serializer [] serdeproto/StorageSerializer - (-serialize-db-root [_ {:keys [t block prevIndex timestamp stats - ledger-id ecount fork forkBlock - spot psot post opst tspo] :as db-root}] + (-serialize-db-root [_ db-root] (reduce-kv (fn [acc k v] (assoc acc (name k) diff --git a/src/fluree/db/storage/core.cljc b/src/fluree/db/storage/core.cljc index 9f68460f1..00390f920 100644 --- a/src/fluree/db/storage/core.cljc +++ b/src/fluree/db/storage/core.cljc @@ -18,16 +18,16 @@ (:serializer conn)) (defn ledger-root-key - [network ledger-id t] - (str network "_" ledger-id "_root_" (util/zero-pad t 15))) + [ledger-alias t] + (str ledger-alias "_root_" (util/zero-pad t 15))) (defn ledger-garbage-prefix - [network ldgr-id] - (str/join "_" [network ldgr-id "garbage"])) + [ledger-alias] + (str/join "_" [ledger-alias "garbage"])) (defn ledger-garbage-key - [network ldgr-id t] - (let [pre (ledger-garbage-prefix network ldgr-id)] + [ledger-alias t] + (let [pre (ledger-garbage-prefix ledger-alias)] (str/join "_" [pre t]))) (defn child-data @@ -74,9 +74,9 @@ "Writes garbage record out for latest index." [db garbage] (go-try - (let [{:keys [conn ledger ledger-id t]} db + (let [{:keys [conn ledger ledger-alias t]} db t' (- t) ;; use positive t integer - data {:ledger-id ledger-id + data {:ledger-alias ledger-alias :t t' :garbage garbage} ser (serdeproto/-serialize-garbage (serde conn) data)] @@ -90,8 +90,8 @@ (let [{:keys [conn ledger commit t ecount stats spot psot post opst tspo fork fork-block]} db t' (- t) - ledger-id (:id commit) - data {:ledger-id ledger-id + ledger-alias (:id commit) + data {:ledger-alias ledger-alias :t t' :ecount (or custom-ecount ecount) :stats (select-keys stats [:flakes :size]) @@ -121,7 +121,7 @@ (defn reify-index-root "Turns each index root node into an unresolved node." - [_conn {:keys [network ledger-id comparators t]} index index-data] + [_conn {:keys [ledger-alias comparators t]} index index-data] (let [cmp (or (get comparators index) (throw (ex-info (str "Internal error reifying db index root: " (pr-str index)) @@ -131,8 +131,7 @@ (:rhs index-data) (update :rhs flake/parts->Flake) (:first index-data) (update :first flake/parts->Flake) true (assoc :comparator cmp - :network network - :ledger-id ledger-id + :ledger-alias ledger-alias :t t :leftmost? true)))) @@ -153,9 +152,9 @@ (defn read-garbage "Returns garbage file data for a given index t." - [conn network ledger-id t] + [conn ledger-alias t] (go-try - (let [key (ledger-garbage-key network ledger-id t) + (let [key (ledger-garbage-key ledger-alias t) data ( branch-metadata From 5a06b27473ee1ba3b66933b5a5fba37f38ac877a Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Thu, 22 Jun 2023 12:29:47 -0400 Subject: [PATCH 08/32] fluree.db.storage.core -> fluree.db.storage --- dev/user.clj | 2 +- src/fluree/db/conn/file.cljc | 2 +- src/fluree/db/conn/ipfs.cljc | 2 +- src/fluree/db/conn/memory.cljc | 2 +- src/fluree/db/conn/s3.clj | 2 +- src/fluree/db/indexer/default.cljc | 2 +- src/fluree/db/json_ld/reify.cljc | 2 +- src/fluree/db/{storage/core.cljc => storage.cljc} | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename src/fluree/db/{storage/core.cljc => storage.cljc} (99%) diff --git a/dev/user.clj b/dev/user.clj index a8cfc0d44..bac062570 100644 --- a/dev/user.clj +++ b/dev/user.clj @@ -10,7 +10,7 @@ [fluree.db.flake :as flake] [fluree.db.util.json :as json] [fluree.db.serde.json :as serdejson] - [fluree.db.storage.core :as storage] + [fluree.db.storage :as storage] [fluree.db.query.fql :as fql] [fluree.db.query.range :as query-range] [fluree.db.dbproto :as dbproto] diff --git a/src/fluree/db/conn/file.cljc b/src/fluree/db/conn/file.cljc index be7919c43..2a3ef3114 100644 --- a/src/fluree/db/conn/file.cljc +++ b/src/fluree/db/conn/file.cljc @@ -10,7 +10,7 @@ [fluree.db.conn.cache :as conn-cache] [fluree.db.conn.state-machine :as state-machine] [fluree.db.util.log :as log :include-macros true] - [fluree.db.storage.core :as storage] + [fluree.db.storage :as storage] [fluree.db.indexer.default :as idx-default] [fluree.db.serde.json :refer [json-serde]] #?@(:cljs [["fs" :as fs] diff --git a/src/fluree/db/conn/ipfs.cljc b/src/fluree/db/conn/ipfs.cljc index 9dbcf6675..1c6702d04 100644 --- a/src/fluree/db/conn/ipfs.cljc +++ b/src/fluree/db/conn/ipfs.cljc @@ -1,5 +1,5 @@ (ns fluree.db.conn.ipfs - (:require [fluree.db.storage.core :as storage] + (:require [fluree.db.storage :as storage] [fluree.db.index :as index] [fluree.db.util.context :as ctx-util] [fluree.db.util.core :as util :refer [exception?]] diff --git a/src/fluree/db/conn/memory.cljc b/src/fluree/db/conn/memory.cljc index a4b06809f..8bf91da78 100644 --- a/src/fluree/db/conn/memory.cljc +++ b/src/fluree/db/conn/memory.cljc @@ -1,6 +1,6 @@ (ns fluree.db.conn.memory (:require [clojure.core.async :as async :refer [go]] - [fluree.db.storage.core :as storage] + [fluree.db.storage :as storage] [fluree.db.index :as index] [fluree.db.util.context :as ctx-util] [fluree.db.util.core :as util] diff --git a/src/fluree/db/conn/s3.clj b/src/fluree/db/conn/s3.clj index 5de9f3d69..7c1368386 100644 --- a/src/fluree/db/conn/s3.clj +++ b/src/fluree/db/conn/s3.clj @@ -12,7 +12,7 @@ [fluree.db.indexer.default :as idx-default] [fluree.db.ledger.proto :as ledger-proto] [fluree.db.serde.json :refer [json-serde]] - [fluree.db.storage.core :as storage] + [fluree.db.storage :as storage] [fluree.db.util.context :as ctx-util] [fluree.db.util.json :as json] [fluree.db.util.log :as log] diff --git a/src/fluree/db/indexer/default.cljc b/src/fluree/db/indexer/default.cljc index f94bebdb6..2396a47f1 100644 --- a/src/fluree/db/indexer/default.cljc +++ b/src/fluree/db/indexer/default.cljc @@ -1,7 +1,7 @@ (ns fluree.db.indexer.default (:require [fluree.db.indexer.proto :as idx-proto] [fluree.db.index :as index] - [fluree.db.storage.core :as storage] + [fluree.db.storage :as storage] [fluree.db.flake :as flake] [fluree.db.util.core :as util #?(:clj :refer :cljs :refer-macros) [try* catch*]] [clojure.core.async :as async] diff --git a/src/fluree/db/json_ld/reify.cljc b/src/fluree/db/json_ld/reify.cljc index 32b4b9202..46b5ed135 100644 --- a/src/fluree/db/json_ld/reify.cljc +++ b/src/fluree/db/json_ld/reify.cljc @@ -6,7 +6,7 @@ [fluree.db.json-ld.vocab :as vocab] [fluree.db.util.async :refer [ Date: Mon, 26 Jun 2023 09:06:44 -0400 Subject: [PATCH 09/32] fix recursive invocation arity --- src/fluree/db/storage.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/storage.cljc b/src/fluree/db/storage.cljc index d845b11a0..071f1b7b4 100644 --- a/src/fluree/db/storage.cljc +++ b/src/fluree/db/storage.cljc @@ -223,7 +223,7 @@ (defn resolve-index-node ([conn node] - (resolve-index-node node nil)) + (resolve-index-node conn node nil)) ([conn {:keys [comparator leaf] :as node} error-fn] (assert comparator "Cannot resolve index node; configuration does not have a comparator.") (let [return-ch (async/chan)] From 33dd70e77e3e7823f6c0e16b83dc8310c2d4bde3 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Tue, 18 Jul 2023 16:50:32 +0200 Subject: [PATCH 10/32] remove persistent-sorted-set dependency --- src/fluree/db/flake.cljc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fluree/db/flake.cljc b/src/fluree/db/flake.cljc index 50d867320..7a3f080b1 100644 --- a/src/fluree/db/flake.cljc +++ b/src/fluree/db/flake.cljc @@ -1,7 +1,6 @@ (ns fluree.db.flake (:refer-clojure :exclude [split-at sorted-set-by sorted-map-by take last]) (:require [clojure.data.avl :as avl] - [me.tonsky.persistent-sorted-set :as pss] [fluree.db.constants :as const] [fluree.db.util.core :as util] #?(:clj [clojure.pprint :as pprint])) From 0323a1ae9eb534b7208b1afefcf23b520a719f66 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Tue, 27 Jun 2023 11:53:20 -0400 Subject: [PATCH 11/32] remove unused functions --- src/fluree/db/query/range.cljc | 72 ---------------------------------- 1 file changed, 72 deletions(-) diff --git a/src/fluree/db/query/range.cljc b/src/fluree/db/query/range.cljc index b3b3af6ef..2f3131710 100644 --- a/src/fluree/db/query/range.cljc +++ b/src/fluree/db/query/range.cljc @@ -367,75 +367,3 @@ (throw e)) range-ch ([idx-range] idx-range))))))) - -(defn non-nil-non-boolean? - [o] - (and (not (nil? o)) - (not (boolean? o)))) - -(defn tag-string? - [possible-tag] - (re-find #"^[a-zA-Z0-9-_]*/[a-zA-Z0-9-_]*:[a-zA-Z0-9-]*$" possible-tag)) - -(def ^:const tag-sid-start (flake/min-subject-id const/$_tag)) -(def ^:const tag-sid-end (flake/max-subject-id const/$_tag)) - -(defn is-tag-flake? - "Returns true if flake is a root setting flake." - [f] - (<= tag-sid-start (flake/o f) tag-sid-end)) - - -(defn coerce-tag-flakes - [db flakes] - (async/go-loop [[flake & r] flakes - acc []] - (if flake - (if (is-tag-flake? flake) - (let [[s p o _ t op m] (flake/Flake->parts flake) - o* (= [(flake/max-subject-id id)] - <= [(flake/min-subject-id id)] - opts)) - (throw (ex-info (str "Invalid collection name: " (pr-str name)) - {:status 400 - :error :db/invalid-collection}))) - (catch* e e))))) - -(defn _block-or_tx-collection - "Returns spot index range for only the requested collection." - [db opts] - (index-range db :spot > [0] <= [util/min-long] opts)) - -(defn txn-from-flakes - "Returns vector of transactions from a set of flakes. - Each transaction is a map with the following keys: - 1. db - the associated ledger - 2. tx - a map containing all transaction data in the original cmd - 3. nonce - the nonce - 4. auth - the authority that submitted the transaction - 5. expire - expiration" - [flakes] - (loop [[flake' & r] flakes result* []] - (if (nil? flake') - result* - (let [obj (flake/o flake') - cmd-map (try* - (json/parse obj) - (catch* e nil)) ; log an error if transaction is not parsable? - {:keys [type db tx nonce auth expire]} cmd-map] - (recur r - (if (= type "tx") - (conj result* {:db db :tx tx :nonce nonce :auth auth :expire expire}) - result*)))))) From 6428114a0110d20dc4dc2071cf12456345b5d43c Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Wed, 28 Jun 2023 16:50:39 -0400 Subject: [PATCH 12/32] remove unused namespace --- src/fluree/db/util/tx.clj | 152 -------------------------------------- 1 file changed, 152 deletions(-) delete mode 100644 src/fluree/db/util/tx.clj diff --git a/src/fluree/db/util/tx.clj b/src/fluree/db/util/tx.clj deleted file mode 100644 index f812ea2c9..000000000 --- a/src/fluree/db/util/tx.clj +++ /dev/null @@ -1,152 +0,0 @@ -(ns fluree.db.util.tx - (:require [fluree.db.util.json :as json] - [fluree.crypto :as crypto] - [clojure.string :as str] - [fluree.db.util.async :refer [ (try (json/parse cmd) - (catch Exception _ - (throw (ex-info (format "Transaction %s is not valid JSON, ignoring." id) - {:status 400 :error :db/invalid-transaction})))) - (assoc :txid id - :cmd cmd - :sig sig - :signed signed) - util/without-nils) - _ (log/trace "Validating command:" cmd-map) - sig-authority (try (crypto/account-id-from-message (or signed cmd) sig) - (catch Exception _ - (throw (ex-info (format "Transaction %s has an invalid signature." id) - {:status 400 :error :db/invalid-signature})))) - ;; merge everything together into one map for transaction. - current-time (System/currentTimeMillis) - {:keys [auth authority expire]} cmd-map - expired? (and expire (< expire current-time)) - _ (when expired? - (throw (ex-info (format "Transaction %s is expired. Current time: %s expire time: %s." id current-time expire) - {:status 400 :error :db/expired-transaction}))) - cmd-map* (cond - (and (nil? auth) (nil? authority)) - (assoc cmd-map :auth sig-authority) - - (and (nil? auth) authority) - (throw (ex-info (format "Transaction %s invalid. An authority without an auth is not allowed." id) - {:status 400 :error :db/missing-auth})) - - (and auth authority) - (if (= authority sig-authority) - cmd-map - (throw (ex-info (format "Transaction %s is invalid. Signing authority: %s does not match command authority: %s." id sig-authority authority) - {:status 400 :error :db/invalid-authority}))) - - (and auth (nil? authority)) - (if (= auth sig-authority) - cmd-map - (assoc cmd-map :authority sig-authority)))] - cmd-map*)) - - -(defn gen-tx-hash - "From a list of transaction flakes, returns the sha3 hash. - - Note, this assumes the _tx/hash flake is NOT included in this list, - else the resulting hash will be different from the one that would have - been computed when performing the transaction." - ([tx-flakes] - ;; sort in block sort order as defined by fluree.db.flake/cmp-flakes-block - (-> (apply flake/sorted-set-by flake/cmp-flakes-block tx-flakes) - (gen-tx-hash true))) - ([tx-flakes sorted?] - (if-not sorted? - (gen-tx-hash tx-flakes) - (->> tx-flakes - (mapv #(let [f %] - (vector (flake/s f) (flake/p f) (flake/o f) (flake/t f) (flake/op f) (flake/m f)))) - (json/stringify) - (crypto/sha3-256))))) - - -;;; -;;; Block merkle root calculation -;;; - -(defn- exp [x n] - (loop [acc 1 n n] - (if (zero? n) - acc - (recur (long (* x acc)) (dec n))))) ; long keeps recur arg primitive - -(defn- find-closest-power-2 - [n] - (loop [i 1] - (if (>= (exp 2 i) n) - (exp 2 i) - (recur (inc i))))) - -(defn- generate-hashes - [cmds] - (loop [[f s & r] cmds - acc []] - (let [hash (crypto/sha2-256 (str f s)) - acc* (conj acc hash)] - (if r - (recur r acc*) - acc*)))) - -(defn generate-merkle-root - "hashes should already be in the correct order." - [& hashes] - (let [count-cmds (count hashes) - repeat-last (- count-cmds (find-closest-power-2 count-cmds)) - leaves-ordrd (concat hashes (repeat repeat-last (last hashes)))] - (loop [merkle-results (apply generate-hashes leaves-ordrd)] - (if (> 1 (count merkle-results)) - (recur (apply generate-hashes merkle-results)) - (first merkle-results))))) - - -;; TODO - moved this from the original transact namespace. Need to look at how this special treatment is handled -;; and verify it is being done in a reasonable way. -(defn create-new-ledger-tx - [tx-map] - (let [{:keys [ledger alias auth doc fork forkBlock]} tx-map - ledger-name (if (sequential? ledger) - (str (first ledger) "/" (second ledger)) - (str/replace ledger "/$" "/")) - tx (util/without-nils - {:_id "db$newdb" - :_action :insert - :id ledger-name - :alias (or alias ledger-name) - :root auth - :doc doc - :fork fork - :forkBlock forkBlock})] - [tx])) - - -(defn make-candidate-db - "Assigns a tempid to all index roots, which ensures caching for this candidate db - is independent from any 'official' db with the same block." - [db] - (let [tempid (random-uuid) - indexes [:spot :psot :post :opst]] - (reduce - (fn [db idx] - (let [index (assoc (get db idx) :tempid tempid)] - (assoc db idx index))) - db indexes))) From f3b397d5ca29085f5414e716792a296008d87516 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Wed, 28 Jun 2023 18:42:53 -0400 Subject: [PATCH 13/32] fix typo --- src/fluree/db/index.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index 85d0b9b8d..ad1f199d0 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -43,7 +43,7 @@ (catch* e (log/error e "Error resolving index node:" - (select-keys node [:id :ledger-aliasd])) + (select-keys node [:id :ledger-alias])) (>! error-ch e))))) (defn resolved? From 95f9a05f2cd713b77bcc59d723090e6da03c6a17 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Wed, 12 Jul 2023 15:54:37 +0200 Subject: [PATCH 14/32] use pre-existing util/sequential fn --- src/fluree/db/json_ld/reify.cljc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fluree/db/json_ld/reify.cljc b/src/fluree/db/json_ld/reify.cljc index 46b5ed135..882a828cd 100644 --- a/src/fluree/db/json_ld/reify.cljc +++ b/src/fluree/db/json_ld/reify.cljc @@ -4,6 +4,7 @@ [fluree.db.constants :as const] [fluree.db.json-ld.ledger :as jld-ledger] [fluree.db.json-ld.vocab :as vocab] + [fluree.db.util.core :as util] [fluree.db.util.async :refer [ Date: Wed, 12 Jul 2023 11:56:55 +0200 Subject: [PATCH 15/32] resolve empty branches too --- src/fluree/db/conn/file.cljc | 2 +- src/fluree/db/conn/ipfs.cljc | 2 +- src/fluree/db/conn/memory.cljc | 2 +- src/fluree/db/conn/s3.clj | 2 +- src/fluree/db/indexer/default.cljc | 2 +- src/fluree/db/storage.cljc | 15 +++++++++++++++ 6 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/fluree/db/conn/file.cljc b/src/fluree/db/conn/file.cljc index 2a3ef3114..d3b7f0318 100644 --- a/src/fluree/db/conn/file.cljc +++ b/src/fluree/db/conn/file.cljc @@ -255,7 +255,7 @@ [conn {:keys [id leaf tempid] :as node}] (let [cache-key [::resolve id tempid]] (if (= :empty id) - (storage/resolve-empty-leaf node) + (storage/resolve-empty-node node) (conn-cache/lru-lookup lru-cache-atom cache-key diff --git a/src/fluree/db/conn/ipfs.cljc b/src/fluree/db/conn/ipfs.cljc index 1c6702d04..392260162 100644 --- a/src/fluree/db/conn/ipfs.cljc +++ b/src/fluree/db/conn/ipfs.cljc @@ -131,7 +131,7 @@ [conn {:keys [id leaf tempid] :as node}] (let [cache-key [::resolve id tempid]] (if (= :empty id) - (storage/resolve-empty-leaf node) + (storage/resolve-empty-node node) (conn-cache/lru-lookup lru-cache-atom cache-key diff --git a/src/fluree/db/conn/memory.cljc b/src/fluree/db/conn/memory.cljc index 8bf91da78..09234a045 100644 --- a/src/fluree/db/conn/memory.cljc +++ b/src/fluree/db/conn/memory.cljc @@ -163,7 +163,7 @@ [_ node] ;; all root index nodes will be empty - (storage/resolve-empty-leaf node)) + (storage/resolve-empty-node node)) #?@(:clj [full-text/IndexConnection diff --git a/src/fluree/db/conn/s3.clj b/src/fluree/db/conn/s3.clj index 7c1368386..db498a475 100644 --- a/src/fluree/db/conn/s3.clj +++ b/src/fluree/db/conn/s3.clj @@ -224,7 +224,7 @@ (resolve [conn {:keys [id leaf tempid] :as node}] (let [cache-key [::resolve id tempid]] (if (= :empty id) - (storage/resolve-empty-leaf node) + (storage/resolve-empty-node node) (conn-cache/lru-lookup lru-cache-atom cache-key (fn [_] (storage/resolve-index-node diff --git a/src/fluree/db/indexer/default.cljc b/src/fluree/db/indexer/default.cljc index 2396a47f1..b56e176a5 100644 --- a/src/fluree/db/indexer/default.cljc +++ b/src/fluree/db/indexer/default.cljc @@ -325,7 +325,7 @@ collection purposes" [{:keys [id] :as node}] (cond-> node - (index/resolved? node) (assoc ::old-id id))) + (index/resolved? node) (assoc ::old-id id))) (defn update-branch-ids "When using IPFS, we don't know what the leaf id will be until written, therefore diff --git a/src/fluree/db/storage.cljc b/src/fluree/db/storage.cljc index 071f1b7b4..c5d9de951 100644 --- a/src/fluree/db/storage.cljc +++ b/src/fluree/db/storage.cljc @@ -251,3 +251,18 @@ empty-node (assoc node :flakes empty-set)] (async/put! ch empty-node) ch)) + +(defn resolve-empty-branch + [{:keys [comparator ledger-alias] :as node}] + (let [ch (async/chan) + child (index/empty-leaf ledger-alias comparator) + children (index/sorted-node-set-by comparator [child]) + empty-node (assoc node :children children)] + (async/put! ch empty-node) + ch)) + +(defn resolve-empty-node + [node] + (if (index/leaf? node) + (resolve-empty-leaf node) + (resolve-empty-branch node))) From a8883fa4f7aa967bf86dc542a7b748f99d3b92d4 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Fri, 14 Jul 2023 19:15:28 +0200 Subject: [PATCH 16/32] don't resolve empty nodes unless necessary --- src/fluree/db/storage.cljc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fluree/db/storage.cljc b/src/fluree/db/storage.cljc index c5d9de951..edf44227e 100644 --- a/src/fluree/db/storage.cljc +++ b/src/fluree/db/storage.cljc @@ -263,6 +263,9 @@ (defn resolve-empty-node [node] - (if (index/leaf? node) - (resolve-empty-leaf node) - (resolve-empty-branch node))) + (if (index/resolved? node) + (doto (async/chan) + (async/put! node)) + (if (index/leaf? node) + (resolve-empty-leaf node) + (resolve-empty-branch node)))) From ac3d736f695b095a6828e981ed9c61adc8f845a0 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Tue, 18 Jul 2023 17:00:20 +0200 Subject: [PATCH 17/32] use sorted map for branch children again --- src/fluree/db/storage.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/storage.cljc b/src/fluree/db/storage.cljc index edf44227e..0caadd660 100644 --- a/src/fluree/db/storage.cljc +++ b/src/fluree/db/storage.cljc @@ -256,7 +256,7 @@ [{:keys [comparator ledger-alias] :as node}] (let [ch (async/chan) child (index/empty-leaf ledger-alias comparator) - children (index/sorted-node-set-by comparator [child]) + children (flake/sorted-map-by comparator child) empty-node (assoc node :children children)] (async/put! ch empty-node) ch)) From 366784bd16b9b17563d3cf51ebafa2a4b52adde0 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Tue, 18 Jul 2023 17:47:24 +0200 Subject: [PATCH 18/32] remove include? argument to tree-chan in favor of a filtering xf --- src/fluree/db/index.cljc | 14 +++++++++----- src/fluree/db/indexer/default.cljc | 2 +- src/fluree/db/query/range.cljc | 12 ++++-------- src/fluree/db/query/subject_crawl/common.cljc | 4 +++- src/fluree/db/query/subject_crawl/subject.cljc | 2 +- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index ad1f199d0..aee261210 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -54,6 +54,11 @@ (leaf? node) (not (nil? (:flakes node))) (branch? node) (not (nil? (:children node))))) +(defn resolved-leaf? + [node] + (and (leaf? node) + (resolved? node))) + (defn unresolve "Clear the populated child node attributes from the supplied `node` map if it represents a branch, or the populated flakes if `node` represents a leaf." @@ -336,9 +341,9 @@ is an optional parameter specifying the number of nodes to load concurrently, and `xf` is an optional transducer that will transform the output stream if supplied." - ([r root resolve? include? error-ch] - (tree-chan r root resolve? include? 1 identity error-ch)) - ([r root resolve? include? n xf error-ch] + ([r root resolve? error-ch] + (tree-chan r root resolve? 1 identity error-ch)) + ([r root resolve? n xf error-ch] (let [out (chan n xf)] (go (let [root-node (! out (unmark-expanded node))) + (do (>! out (unmark-expanded node)) (recur stack*)) (let [children ( stack* diff --git a/src/fluree/db/indexer/default.cljc b/src/fluree/db/indexer/default.cljc index b56e176a5..fb1466c0b 100644 --- a/src/fluree/db/indexer/default.cljc +++ b/src/fluree/db/indexer/default.cljc @@ -402,7 +402,7 @@ novel? (fn [node] (or (seq remove-preds) (seq (index/novelty-subrange node t novelty))))] - (->> (index/tree-chan conn root novel? (constantly true) 1 refresh-xf error-ch) + (->> (index/tree-chan conn root novel? 1 refresh-xf error-ch) (write-resolved-nodes db idx changes-ch error-ch)))) (defn extract-root diff --git a/src/fluree/db/query/range.cljc b/src/fluree/db/query/range.cljc index 2f3131710..dd182a0ae 100644 --- a/src/fluree/db/query/range.cljc +++ b/src/fluree/db/query/range.cljc @@ -92,11 +92,6 @@ m' (or m (if (identical? >= test) util/min-integer util/max-integer))] (flake/create s p o' dt t op m'))) -(defn resolved-leaf? - [node] - (and (index/leaf? node) - (index/resolved? node))) - (defn intersects-range? "Returns true if the supplied `node` contains flakes between the `lower` and `upper` flakes, according to the `node`'s comparator." @@ -129,7 +124,8 @@ (let [flake-xfs (cond-> [(query-filter opts)] flake-xf (conj flake-xf)) flake-xf* (apply comp flake-xfs) - query-xf (comp (map :flakes) + query-xf (comp (filter index/resolved-leaf?) + (map :flakes) (map (fn [flakes] (flake/subrange flakes start-test start-flake @@ -150,7 +146,7 @@ in-range? (fn [node] (intersects-range? node range-set)) query-xf (extract-query-flakes opts)] - (index/tree-chan resolver root in-range? resolved-leaf? 1 query-xf error-ch))) + (index/tree-chan resolver root in-range? 1 query-xf error-ch))) (defn unauthorized? [f] @@ -292,7 +288,7 @@ :end-test end-test :end-flake end-flake})] (go-try - (let [history-ch (->> (index/tree-chan resolver idx-root in-range? resolved-leaf? 1 query-xf error-ch) + (let [history-ch (->> (index/tree-chan resolver idx-root in-range? 1 query-xf error-ch) (filter-authorized db start-flake end-flake error-ch) (into-page limit offset flake-limit))] (async/alt! diff --git a/src/fluree/db/query/subject_crawl/common.cljc b/src/fluree/db/query/subject_crawl/common.cljc index 94b84400a..20509aa5d 100644 --- a/src/fluree/db/query/subject_crawl/common.cljc +++ b/src/fluree/db/query/subject_crawl/common.cljc @@ -2,6 +2,7 @@ (:require [clojure.core.async :refer [go] :as async] [fluree.db.util.async :refer [ [(map :flakes) + (apply comp (cond-> [(filter index/resolved-leaf?) + (map :flakes) (map (fn [flakes] (flake/subrange flakes start-test start-flake diff --git a/src/fluree/db/query/subject_crawl/subject.cljc b/src/fluree/db/query/subject_crawl/subject.cljc index 84c0d6644..dc140d9e7 100644 --- a/src/fluree/db/query/subject_crawl/subject.cljc +++ b/src/fluree/db/query/subject_crawl/subject.cljc @@ -55,7 +55,7 @@ (map (fn [flakes] (filter filter-fn flakes))))}) resolver (index/->CachedTRangeResolver conn (get novelty idx*) t t (:lru-cache-atom conn)) - tree-chan (index/tree-chan resolver idx-root in-range? query-range/resolved-leaf? 1 query-xf error-ch)] + tree-chan (index/tree-chan resolver idx-root in-range? 1 query-xf error-ch)] (async/go-loop [] (let [next-chunk ( Date: Wed, 19 Jul 2023 11:02:34 +0200 Subject: [PATCH 19/32] add min/max values for flake components --- src/fluree/db/flake.cljc | 15 +++++++++++++-- src/fluree/db/query/range.cljc | 12 ++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/fluree/db/flake.cljc b/src/fluree/db/flake.cljc index 7a3f080b1..6048f2969 100644 --- a/src/fluree/db/flake.cljc +++ b/src/fluree/db/flake.cljc @@ -61,6 +61,18 @@ (def ^:const MIN-PREDICATE-ID (min-subject-id const/$_predicate)) (def ^:const MAX-PREDICATE-ID (max-subject-id const/$_predicate)) +(def ^:const min-s util/max-long) +(def ^:const max-s util/min-long) +(def ^:const min-p 0) +(def ^:const max-p MAX-PREDICATE-ID) +(def ^:const min-dt util/min-integer) +(def ^:const max-dt util/max-integer) +(def ^:const min-t 0) +(def ^:const max-t util/min-long) +(def ^:const min-op false) +(def ^:const max-op true) +(def ^:const min-meta util/min-integer) +(def ^:const max-meta util/max-integer) (defn sid->cid "Will return a collection id from a subject-id." @@ -228,8 +240,7 @@ [(s flake) (p flake) (o flake) (dt flake) (t flake) (op flake) (m flake)]) (def maximum - "The largest flake possible" - (->Flake util/max-long 0 util/max-long const/$xsd:decimal 0 true nil)) + (->Flake max-s max-p max-s max-dt max-t max-op max-meta)) (defn- assoc-flake "Assoc for Flakes" diff --git a/src/fluree/db/query/range.cljc b/src/fluree/db/query/range.cljc index dd182a0ae..d40421713 100644 --- a/src/fluree/db/query/range.cljc +++ b/src/fluree/db/query/range.cljc @@ -47,12 +47,12 @@ :tspo [p2 (coerce-predicate db p3) p4 p1 op m]))) -(def ^{:private true :const true} subject-min-match [util/max-long]) -(def ^{:private true :const true} subject-max-match [util/min-long]) -(def ^{:private true :const true} pred-min-match [0]) -(def ^{:private true :const true} pred-max-match [flake/MAX-PREDICATE-ID]) -(def ^{:private true :const true} txn-max-match [util/min-long]) -(def ^{:private true :const true} txn-min-match [0]) +(def ^{:private true :const true} subject-min-match [flake/min-s]) +(def ^{:private true :const true} subject-max-match [flake/max-s]) +(def ^{:private true :const true} pred-min-match [flake/min-p]) +(def ^{:private true :const true} pred-max-match [flake/max-p]) +(def ^{:private true :const true} txn-max-match [flake/min-t]) +(def ^{:private true :const true} txn-min-match [flake/max-t]) (defn- min-match From d9bcd0205b4a8a5bc37dbdbc8fa1bc0fafd174ce Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Fri, 21 Jul 2023 13:07:30 +0200 Subject: [PATCH 20/32] ensure child maps are *always* avl maps --- src/fluree/db/json_ld/commit_data.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/json_ld/commit_data.cljc b/src/fluree/db/json_ld/commit_data.cljc index c86b4f284..8e5bd4c8d 100644 --- a/src/fluree/db/json_ld/commit_data.cljc +++ b/src/fluree/db/json_ld/commit_data.cljc @@ -424,7 +424,7 @@ children* (reduce-kv (fn [children* k v] (assoc children* k (assoc v :tt-id tt-id))) - {} children)] + (empty children) children)] (assoc db* idx (assoc node :tt-id tt-id :children children*)))) db indexes) From 90582b2ed15e5d19bdb291c7a3729262a3ce5a88 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Fri, 21 Jul 2023 13:45:13 +0200 Subject: [PATCH 21/32] use transients to add/remove flakes to novelty sets for perfermance --- src/fluree/db/flake.cljc | 7 +++++++ src/fluree/db/json_ld/commit_data.cljc | 15 +++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/fluree/db/flake.cljc b/src/fluree/db/flake.cljc index 6048f2969..ce9ba3109 100644 --- a/src/fluree/db/flake.cljc +++ b/src/fluree/db/flake.cljc @@ -560,6 +560,13 @@ [ss to-remove] (transient-reduce disj! ss to-remove)) +(defn revise + [ss to-add to-remove] + (as-> (transient ss) trans + (reduce disj! trans to-remove) + (reduce conj! trans to-add) + (persistent! trans))) + (defn assoc-all [sm entries] (transient-reduce (fn [m [k v]] diff --git a/src/fluree/db/json_ld/commit_data.cljc b/src/fluree/db/json_ld/commit_data.cljc index 8e5bd4c8d..2caaa07d1 100644 --- a/src/fluree/db/json_ld/commit_data.cljc +++ b/src/fluree/db/json_ld/commit_data.cljc @@ -364,11 +364,6 @@ [flakes] (filter ref? flakes)) -(defn update-novelty-idx - [novelty-idx add remove] - (-> (reduce disj novelty-idx remove) - (into add))) - (defn update-novelty ([db add] (update-novelty db add [])) @@ -383,11 +378,11 @@ add (+ (flake/size-bytes add)) rem (- (flake/size-bytes rem)))] (-> db - (update-in [:novelty :spot] update-novelty-idx add rem) - (update-in [:novelty :psot] update-novelty-idx add rem) - (update-in [:novelty :post] update-novelty-idx add rem) - (update-in [:novelty :opst] update-novelty-idx ref-add ref-rem) - (update-in [:novelty :tspo] update-novelty-idx add rem) + (update-in [:novelty :spot] flake/revise add rem) + (update-in [:novelty :psot] flake/revise add rem) + (update-in [:novelty :post] flake/revise add rem) + (update-in [:novelty :opst] flake/revise ref-add ref-rem) + (update-in [:novelty :tspo] flake/revise add rem) (update-in [:novelty :size] + flake-size) (update-in [:stats :size] + flake-size) (update-in [:stats :flakes] + flake-count))))) From d6a1f7dd31d044e9db5c6c83b43ecd4ea3d29022 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Fri, 21 Jul 2023 13:52:48 +0200 Subject: [PATCH 22/32] cleanup ns requires --- src/fluree/db/query/range.cljc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/fluree/db/query/range.cljc b/src/fluree/db/query/range.cljc index d40421713..cb4cc4bcb 100644 --- a/src/fluree/db/query/range.cljc +++ b/src/fluree/db/query/range.cljc @@ -4,11 +4,10 @@ [fluree.db.index :as index] [fluree.db.util.schema :as schema-util] [fluree.db.util.core :as util #?(:clj :refer :cljs :refer-macros) [try* catch*]] - [fluree.db.util.json :as json] [fluree.db.util.log :as log :include-macros true] [fluree.db.flake :as flake] - #?(:clj [clojure.core.async :refer [chan go go-loop !] :as async] - :cljs [cljs.core.async :refer [chan !] :refer-macros [go go-loop] :as async]) + #?(:clj [clojure.core.async :refer [chan go >!] :as async] + :cljs [cljs.core.async :refer [chan >!] :refer-macros [go] :as async]) [fluree.db.permissions-validate :as perm-validate] [fluree.db.util.async :refer [ Date: Fri, 21 Jul 2023 13:53:04 +0200 Subject: [PATCH 23/32] add minimum flake for consistency with max; add nearest ss fn --- src/fluree/db/flake.cljc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/fluree/db/flake.cljc b/src/fluree/db/flake.cljc index ce9ba3109..a0e77d6f9 100644 --- a/src/fluree/db/flake.cljc +++ b/src/fluree/db/flake.cljc @@ -242,6 +242,9 @@ (def maximum (->Flake max-s max-p max-s max-dt max-t max-op max-meta)) +(def minimum + (->Flake min-s min-p min-s min-dt min-t min-op min-meta)) + (defn- assoc-flake "Assoc for Flakes" [flake k v] @@ -506,6 +509,10 @@ (avl/subrange ss start-test start-flake end-test end-flake))) +(defn nearest + [ss test f] + (avl/nearest ss test f)) + (defn split-at [n ss] (avl/split-at n ss)) From 8366750813821874ad56eaf4451f71dd723dcb20 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Fri, 21 Jul 2023 13:54:10 +0200 Subject: [PATCH 24/32] add start/end flakes to tree-chan; trim resolved nodes based on them --- src/fluree/db/index.cljc | 80 +++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 17 deletions(-) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index aee261210..bbc63ee65 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -36,16 +36,6 @@ "Populate the supplied index branch or leaf node maps with either the child node attributes or the flakes they store, respectively.")) -(defn try-resolve - [r error-ch node] - (go - (try* (! error-ch e))))) - (defn resolved? "Returns `true` if the data associated with the index node map `node` is fully resolved from storage" @@ -314,20 +304,69 @@ [node] (-> node ::expanded true?)) +(defn trim-leaf + [leaf start-flake end-flake] + (cond + (and start-flake end-flake) + (update leaf :flakes flake/subrange >= start-flake <= end-flake) + + start-flake + (update leaf :flakes flake/subrange >= start-flake) + + end-flake + (update leaf :flakes flake/subrange <= end-flake) + + :else + leaf)) + +(defn trim-branch + [{:keys [children] :as branch} start-flake end-flake] + (let [start-key (some->> start-flake (flake/nearest children <=) key) + end-key (some->> end-flake (flake/nearest children <=) key)] + (cond + (and start-key end-key) + (update branch :children flake/subrange >= start-key <= end-key) + + start-key + (update branch :children flake/subrange >= start-key) + + end-key + (update branch :children flake/subrange <= end-key) + + :else + branch))) + +(defn trim-node + [node start-flake end-flake] + (if (leaf? node) + (trim-leaf node start-flake end-flake) + (trim-branch node start-flake end-flake))) + +(defn try-resolve + [r start-flake end-flake error-ch node] + (go + (try* (let [resolved (! error-ch e))))) + (defn resolve-when - [r resolve? error-ch node] + [r start-flake end-flake resolve? error-ch node] (if (resolve? node) - (try-resolve r error-ch node) + (try-resolve r start-flake end-flake error-ch node) (doto (chan) (async/put! node)))) (defn resolve-children-when - [r resolve? error-ch branch] + [r start-flake end-flake resolve? error-ch branch] (if (resolved? branch) (->> branch :children (map (fn [[_ child]] - (resolve-when r resolve? error-ch child))) + (resolve-when r start-flake end-flake resolve? error-ch child))) (async/map vector)) (go []))) @@ -344,9 +383,12 @@ ([r root resolve? error-ch] (tree-chan r root resolve? 1 identity error-ch)) ([r root resolve? n xf error-ch] + (tree-chan r root nil nil resolve? n xf error-ch)) + ([r root start-flake end-flake resolve? n xf error-ch] (let [out (chan n xf)] (go - (let [root-node (! out (unmark-expanded node)) (recur stack*)) - (let [children ( stack* (conj (mark-expanded node)) - (into (rseq children)))] + (into (rseq children)))] ; reverse children + ; to ensure final + ; nodes are in + ; depth-first order (recur stack**)))))) (async/close! out))) out))) From 969638e4f1f46f3d6791fd992ea33099060da137 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Fri, 21 Jul 2023 13:54:54 +0200 Subject: [PATCH 25/32] rely on tree-chan to trim resolved nodes instead of with an xf --- src/fluree/db/query/range.cljc | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/fluree/db/query/range.cljc b/src/fluree/db/query/range.cljc index cb4cc4bcb..8b21b7b7a 100644 --- a/src/fluree/db/query/range.cljc +++ b/src/fluree/db/query/range.cljc @@ -119,16 +119,12 @@ map. The result of the transformation will be a stream of collections of flakes from the leaf nodes in the input stream, with one flake collection for each input leaf." - [{:keys [start-flake start-test end-flake end-test flake-xf] :as opts}] + [{:keys [flake-xf] :as opts}] (let [flake-xfs (cond-> [(query-filter opts)] flake-xf (conj flake-xf)) flake-xf* (apply comp flake-xfs) query-xf (comp (filter index/resolved-leaf?) (map :flakes) - (map (fn [flakes] - (flake/subrange flakes - start-test start-flake - end-test end-flake))) (map (fn [flakes] (into [] flake-xf* flakes))))] query-xf)) @@ -140,12 +136,8 @@ [{:keys [lru-cache-atom] :as conn} root novelty error-ch {:keys [from-t to-t start-flake end-flake] :as opts}] (let [resolver (index/->CachedTRangeResolver conn novelty from-t to-t lru-cache-atom) - cmp (:comparator root) - range-set (flake/sorted-set-by cmp start-flake end-flake) - in-range? (fn [node] - (intersects-range? node range-set)) query-xf (extract-query-flakes opts)] - (index/tree-chan resolver root in-range? 1 query-xf error-ch))) + (index/tree-chan resolver root start-flake end-flake any? 1 query-xf error-ch))) (defn unauthorized? [f] @@ -287,7 +279,8 @@ :end-test end-test :end-flake end-flake})] (go-try - (let [history-ch (->> (index/tree-chan resolver idx-root in-range? 1 query-xf error-ch) + (let [history-ch (->> (index/tree-chan resolver idx-root start-flake end-flake + in-range? 1 query-xf error-ch) (filter-authorized db start-flake end-flake error-ch) (into-page limit offset flake-limit))] (async/alt! From e8b737ffe35c3e404cac97cac624e91ea7db7b05 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Fri, 21 Jul 2023 13:59:32 +0200 Subject: [PATCH 26/32] bump up reindexing parallelism because why not? If this eats up too much memory, we can bump it back down. It's pretty sloooooow right now because it's running single threaded. --- src/fluree/db/indexer/default.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/indexer/default.cljc b/src/fluree/db/indexer/default.cljc index fb1466c0b..5845a3017 100644 --- a/src/fluree/db/indexer/default.cljc +++ b/src/fluree/db/indexer/default.cljc @@ -402,7 +402,7 @@ novel? (fn [node] (or (seq remove-preds) (seq (index/novelty-subrange node t novelty))))] - (->> (index/tree-chan conn root novel? 1 refresh-xf error-ch) + (->> (index/tree-chan conn root novel? 4 refresh-xf error-ch) (write-resolved-nodes db idx changes-ch error-ch)))) (defn extract-root From 6de6c2beccad3edf032464e4dc151afafe8d1fe1 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Fri, 21 Jul 2023 14:37:49 +0200 Subject: [PATCH 27/32] add/update docstrings --- src/fluree/db/flake.cljc | 2 ++ src/fluree/db/index.cljc | 21 ++++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/fluree/db/flake.cljc b/src/fluree/db/flake.cljc index a0e77d6f9..4d48d4883 100644 --- a/src/fluree/db/flake.cljc +++ b/src/fluree/db/flake.cljc @@ -568,6 +568,8 @@ (transient-reduce disj! ss to-remove)) (defn revise + "Changes the composition of the sorted set `ss` by adding all the flakes in the + `to-add` collection and removing all flakes in the `to-remove` collection." [ss to-add to-remove] (as-> (transient ss) trans (reduce disj! trans to-remove) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index bbc63ee65..058741a73 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -305,6 +305,9 @@ (-> node ::expanded true?)) (defn trim-leaf + "Remove flakes from the index leaf node `leaf` that are outside of the interval + defined by `start-flake` and `end-flake`. nil values for either `start-flake` + or `end-flake` makes that side of the interval unlimited." [leaf start-flake end-flake] (cond (and start-flake end-flake) @@ -320,6 +323,10 @@ leaf)) (defn trim-branch + "Remove child nodes from the index branch node `branch` that do not contain + flakes in the interval defined by `start-flake` and `end-flake`. nil values + for either `start-flake` or `end-flake` makes that side of the interval + unlimited." [{:keys [children] :as branch} start-flake end-flake] (let [start-key (some->> start-flake (flake/nearest children <=) key) end-key (some->> end-flake (flake/nearest children <=) key)] @@ -337,6 +344,10 @@ branch))) (defn trim-node + "Remove flakes or children from the index leaf or branch node `node` that are + outside of the interval defined by `start-flake` and `end-flake`. nil values + for either `start-flake` or `end-flake` makes that side of the interval + unlimited." [node start-flake end-flake] (if (leaf? node) (trim-leaf node start-flake end-flake) @@ -375,11 +386,11 @@ descended from `root` in depth-first order. `resolve?` is a boolean function that will be applied to each node to determine whether or not the data associated with that node will be resolved from disk using the supplied - `Resolver` `r`. `include?` is a boolean function that will be applied to each - node to determine if it will be included in the final output node stream, `n` - is an optional parameter specifying the number of nodes to load concurrently, - and `xf` is an optional transducer that will transform the output stream if - supplied." + `Resolver` `r`. `start-flake` and `end-flake` are flakes for which only nodes + that contain flakes within the interval defined by them will be considered, + `n` is an optional parameter specifying the number of nodes to load + concurrently, and `xf` is an optional transducer that will transform the + output stream if supplied." ([r root resolve? error-ch] (tree-chan r root resolve? 1 identity error-ch)) ([r root resolve? n xf error-ch] From 7f5e626f49c3035b418ef34cbdcf13e7996045b6 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Fri, 21 Jul 2023 18:02:46 +0200 Subject: [PATCH 28/32] remove unused fns --- src/fluree/db/flake.cljc | 87 ---------------------------------------- 1 file changed, 87 deletions(-) diff --git a/src/fluree/db/flake.cljc b/src/fluree/db/flake.cljc index 4d48d4883..8d5dbe8e2 100644 --- a/src/fluree/db/flake.cljc +++ b/src/fluree/db/flake.cljc @@ -431,28 +431,6 @@ (cmp-bool (op f1) (op f2)) (cmp-meta (m f1) (m f2)))) - -(defn cmp-flakes-history - "Note this is not suitable for a set, only a vector/list." - [f1 f2] - (combine-cmp - (cmp-long (t f1) (t f2)) - #?(:clj (Boolean/compare (op f2) (op f1)) - :cljs (compare (op f2) (op f1))))) - - -(defn cmp-history-quick-reverse-sort - "Sorts by transaction time in ascending order (newest first), then by - the boolean operation descending so assertions (true) come before retractions (false) - so that we can 're-play' the log in reverse order to come up with historical states. - Suitable only for sorting a vector, not a sorted set." - [f1 f2] - (combine-cmp - (cmp-long (t f1) (t f2)) - #?(:clj (Boolean/compare (op f2) (op f1)) - :cljs (compare (op f2) (op f1))))) - - (defn flip-flake "Takes a flake and returns one with the provided block and op flipped from true/false. Don't over-ride no-history, even if no-history for this predicate has changed. New inserts @@ -462,35 +440,6 @@ ([flake t] (->Flake (s flake) (p flake) (o flake) (dt flake) t (not (op flake)) (m flake)))) -(defn slice - "From and to are Flakes" - [ss from to] - (cond - (and from to) (avl/subrange ss >= from <= to) - (nil? from) (avl/subrange ss <= to) - (nil? to) (avl/subrange ss >= from) - :else (throw (ex-info "Unexpected error performing slice, both from and to conditions are nil. Please report." - {:status 500 - :error :db/unexpected-error})))) - -(defn match-spot - "Returns all matching flakes to a specific subject, and optionaly also a predicate if provided - Must be provided with subject/predicate integer ids, no lookups are performed." - [ss sid pid] - (if pid - (avl/subrange ss >= (->Flake sid pid nil -1 nil nil nil) - <= (->Flake sid (inc pid) nil util/max-long nil nil nil)) - (avl/subrange ss > (->Flake (inc sid) MAX-COLL-SUBJECTS nil nil nil nil nil) - < (->Flake (dec sid) -1 nil nil nil nil nil)))) - - -(defn match-post - "Returns all matching flakes to a predicate + object match." - [ss pid o dt] - (avl/subrange ss - >= (->Flake util/max-long pid o dt nil nil nil) - <= (->Flake 0 pid o dt nil nil nil))) - (defn match-tspo "Returns all matching flakes to a specific 't' value." [ss t] @@ -498,25 +447,16 @@ >= (->Flake util/max-long nil nil nil t nil nil) <= (->Flake util/min-long nil nil nil t nil nil))) -(defn lookup - [ss start-flake end-flake] - (avl/subrange ss >= start-flake <= end-flake)) - (defn subrange ([ss test flake] (avl/subrange ss test flake)) ([ss start-test start-flake end-test end-flake] (avl/subrange ss start-test start-flake end-test end-flake))) - (defn nearest [ss test f] (avl/nearest ss test f)) -(defn split-at - [n ss] - (avl/split-at n ss)) - (defn lower-than-all? [f ss] (let [[lower e _] (avl/split-key f ss)] @@ -529,14 +469,6 @@ (and (nil? e) (empty? upper)))) -(defn split-by-flake - "Splits a sorted set at a given flake. If there is an exact match for flake, - puts it in the left-side. Primarily for use with last-flake." - [f ss] - (let [[l e r] (avl/split-key f ss)] - [(if e (conj l e) l) r])) - - (defn sorted-set-by [comparator & flakes] (apply avl/sorted-set-by comparator flakes)) @@ -634,22 +566,3 @@ "Returns approx number of bytes in a collection of flakes." [flakes] (reduce (fn [size f] (+ size (size-flake f))) 0 flakes)) - - -(defn size-kb - "Like size-bytes, but kb. - Rounds down for simplicity, as bytes is just an estimate anyhow." - [flakes] - (-> (size-bytes flakes) - (/ 1000) - (double) - (Math/round))) - - -(defn take - "Takes n flakes from a sorted flake set, retaining the set itself." - [n flake-set] - (if (>= n (count flake-set)) - flake-set - (let [k (nth flake-set n)] - (first (avl/split-key k flake-set))))) From 1b38569002a70cf7ad4e94ac3df3e4b2d7764663 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Mon, 24 Jul 2023 22:20:48 +0200 Subject: [PATCH 29/32] use loops instead of reduce for performance --- src/fluree/db/flake.cljc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/fluree/db/flake.cljc b/src/fluree/db/flake.cljc index 8d5dbe8e2..bdccefe30 100644 --- a/src/fluree/db/flake.cljc +++ b/src/fluree/db/flake.cljc @@ -503,9 +503,15 @@ "Changes the composition of the sorted set `ss` by adding all the flakes in the `to-add` collection and removing all flakes in the `to-remove` collection." [ss to-add to-remove] - (as-> (transient ss) trans - (reduce disj! trans to-remove) - (reduce conj! trans to-add) + (let [trans (transient ss)] + (loop [[f & r] to-remove] + (when f + (disj! trans f) + (recur r))) + (loop [[f & r] to-add] + (when f + (conj! trans f) + (recur r))) (persistent! trans))) (defn assoc-all From a589708ea5e916febbbc89e0026a6f94489666ad Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Mon, 24 Jul 2023 22:21:33 +0200 Subject: [PATCH 30/32] use 1 for buffersize We should parameterize this later --- src/fluree/db/indexer/default.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/indexer/default.cljc b/src/fluree/db/indexer/default.cljc index 5845a3017..fb1466c0b 100644 --- a/src/fluree/db/indexer/default.cljc +++ b/src/fluree/db/indexer/default.cljc @@ -402,7 +402,7 @@ novel? (fn [node] (or (seq remove-preds) (seq (index/novelty-subrange node t novelty))))] - (->> (index/tree-chan conn root novel? 4 refresh-xf error-ch) + (->> (index/tree-chan conn root novel? 1 refresh-xf error-ch) (write-resolved-nodes db idx changes-ch error-ch)))) (defn extract-root From 6ef7eaaffa32a95e8fd124508301cfb52196c60e Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Mon, 24 Jul 2023 22:22:36 +0200 Subject: [PATCH 31/32] correct indentation --- src/fluree/db/index.cljc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fluree/db/index.cljc b/src/fluree/db/index.cljc index 058741a73..cc51477ed 100644 --- a/src/fluree/db/index.cljc +++ b/src/fluree/db/index.cljc @@ -359,10 +359,10 @@ (try* (let [resolved (! error-ch e))))) + (log/error e + "Error resolving index node:" + (select-keys node [:id :ledger-alias])) + (>! error-ch e))))) (defn resolve-when [r start-flake end-flake resolve? error-ch node] From d98579f3e40e3b8a9e46ec08fdb6489e63e363f4 Mon Sep 17 00:00:00 2001 From: Benjamin Lamothe Date: Mon, 24 Jul 2023 22:44:18 +0200 Subject: [PATCH 32/32] change function def to please linter --- src/fluree/db/flake.cljc | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/fluree/db/flake.cljc b/src/fluree/db/flake.cljc index bdccefe30..cb191d06a 100644 --- a/src/fluree/db/flake.cljc +++ b/src/fluree/db/flake.cljc @@ -503,16 +503,18 @@ "Changes the composition of the sorted set `ss` by adding all the flakes in the `to-add` collection and removing all flakes in the `to-remove` collection." [ss to-add to-remove] - (let [trans (transient ss)] - (loop [[f & r] to-remove] - (when f - (disj! trans f) - (recur r))) - (loop [[f & r] to-add] - (when f - (conj! trans f) - (recur r))) - (persistent! trans))) + (let [trans (transient ss) + removed (loop [[f & r] to-remove + t-set trans] + (if f + (recur r (disj! t-set f)) + t-set)) + added (loop [[f & r] to-add + t-set removed] + (if f + (recur r (conj! t-set f)) + t-set))] + (persistent! added))) (defn assoc-all [sm entries]