Skip to content

Commit

Permalink
history query perf improvements
Browse files Browse the repository at this point in the history
Using `group-by` instead of `sort-by` + `partition-by` can produce the assert and
retract flakes in one traversal instead of two. I added an extra subject in the test
suite to make sure that the final results still have a stable sort, and it appears that
they do - we don't care about the specific ordering, just that it's stable.

Also used <! instead of <? because the assert/retract chans won't ever get an error on
them, so the extra error handling isn't necessary.
  • Loading branch information
dpetran committed Feb 2, 2023
1 parent a389c8b commit 639a460
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 16 deletions.
26 changes: 14 additions & 12 deletions src/fluree/db/api/query.cljc
Original file line number Diff line number Diff line change
Expand Up @@ -95,20 +95,22 @@
(defn t-flakes->json-ld
[db compact cache fuel error-ch t-flakes]
(go-try
(let [assert-flakes (not-empty (filter flake/op t-flakes))
retract-flakes (not-empty (filter (complement flake/op) t-flakes))
(let [{assert-flakes true
retract-flakes false} (group-by flake/op t-flakes)

s-asserts-ch (->> (sort-by flake/s assert-flakes)
(partition-by flake/s)
(async/to-chan!))
s-retracts-ch (->> (sort-by flake/s retract-flakes)
(partition-by flake/s)
(async/to-chan!))
s-flake-partitions (fn [flakes]
(->> flakes
(group-by flake/s)
(vals)
(async/to-chan!)))

s-asserts-out-ch (async/chan)
s-asserts-ch (s-flake-partitions assert-flakes)
s-retracts-ch (s-flake-partitions retract-flakes)

s-asserts-out-ch (async/chan)
s-retracts-out-ch (async/chan)

s-asserts-json-ch (async/into [] s-asserts-out-ch)
s-asserts-json-ch (async/into [] s-asserts-out-ch)
s-retracts-json-ch (async/into [] s-retracts-out-ch)]
;; process asserts
(async/pipeline-async 2
Expand All @@ -125,8 +127,8 @@
(async/pipe ch)))
s-retracts-ch)
{(json-ld/compact const/iri-t compact) (- (flake/t (first t-flakes)))
(json-ld/compact const/iri-assert compact) (<? s-asserts-json-ch)
(json-ld/compact const/iri-retract compact) (<? s-retracts-json-ch)})))
(json-ld/compact const/iri-assert compact) (async/<! s-asserts-json-ch)
(json-ld/compact const/iri-retract compact) (async/<! s-retracts-json-ch)})))

(defn history-flakes->json-ld
[db q flakes]
Expand Down
12 changes: 8 additions & 4 deletions test/fluree/db/query/history_test.clj
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,12 @@
:ex/y "bar-3"})

ts3 (-> db3 :commit :time)
db4 @(test-utils/transact ledger {:id :ex/cat
:ex/x "foo-cat"
:ex/y "bar-cat"})
db4 @(test-utils/transact ledger [{:id :ex/cat
:ex/x "foo-cat"
:ex/y "bar-cat"}
{:id :ex/dog
:ex/x "foo-dog"
:ex/y "bar-dog"}])
db5 @(test-utils/transact ledger {:id :ex/dan
:ex/x "foo-cat"
:ex/y "bar-cat"})]
Expand Down Expand Up @@ -74,7 +77,8 @@
:f/assert [{:ex/x "foo-cat" :id :ex/dan}]
:f/retract [{:ex/x "foo-3" :id :ex/dan}]}
{:f/t 4
:f/assert [{:ex/x "foo-cat" :id :ex/cat}]
:f/assert [{:ex/x "foo-dog" :id :ex/dog}
{:ex/x "foo-cat" :id :ex/cat}]
:f/retract []}
{:f/t 3
:f/assert [{:ex/x "foo-3" :id :ex/dan}]
Expand Down

0 comments on commit 639a460

Please sign in to comment.