From 0060e087e12a84f286585a87f7b8d5e984419e8b Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 14 Feb 2023 11:53:10 -0700 Subject: [PATCH 1/9] Add section on running specific tests to README --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 28e00d8ec..3929f9ec3 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,20 @@ NOTE: use `make -j` to run tasks in parallel. * `make nodejs-test` - run node package in node * `make browser-test` - run browser package in headless Chrome +#### Running specific tests + +> This applies to CLJ tests only, not CLJS. + +`clojure -X:cljtest :kaocha.filter/focus [focus-spec]` + +...where `focus-spec` can be a test namespace or a fully-qualified `deftest` +var. Note that the square brackets around the `focus-spec` must be present in +the command, they are NOT there to indicate "optional" or "placeholder" in the +example. + +This feature comes from the test runner kaocha which has +[additional features](https://cljdoc.org/d/lambdaisland/kaocha/1.77.1236/doc/6-focusing-and-skipping). + ### CLJS REPL In order to get a Node or web browser CLJS REPL running, you need to do the From 12b59bca2960625d7873b5ec15c777515907b5e0 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 14 Feb 2023 12:15:54 -0700 Subject: [PATCH 2/9] Fix nested node commits Closes issue #380 --- src/fluree/db/json_ld/commit.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/json_ld/commit.cljc b/src/fluree/db/json_ld/commit.cljc index 683fe04e7..a86f98f75 100644 --- a/src/fluree/db/json_ld/commit.cljc +++ b/src/fluree/db/json_ld/commit.cljc @@ -67,7 +67,7 @@ (if p-flakes (let [fflake (first p-flakes) p-iri ( Date: Tue, 14 Feb 2023 12:16:37 -0700 Subject: [PATCH 3/9] Retry a test assertion w/ a race condition --- test/fluree/db/json_ld/api_test.cljc | 2 +- test/fluree/db/test_utils.cljc | 41 ++++++++++++++++++---------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/test/fluree/db/json_ld/api_test.cljc b/test/fluree/db/json_ld/api_test.cljc index 25eb5e3fe..0f247d2ff 100644 --- a/test/fluree/db/json_ld/api_test.cljc +++ b/test/fluree/db/json_ld/api_test.cljc @@ -34,7 +34,7 @@ :type :schema/Person :schema/fname "Me"}])] @(fluree/commit! ledger db) - (is @(fluree/exists? conn ledger-alias)) + (is (test-utils/retry-exists? conn ledger-alias 10)) (is (not @(fluree/exists? conn "notaledger")))) :cljs diff --git a/test/fluree/db/test_utils.cljc b/test/fluree/db/test_utils.cljc index d48552614..be8560841 100644 --- a/test/fluree/db/test_utils.cljc +++ b/test/fluree/db/test_utils.cljc @@ -107,20 +107,31 @@ (->> @(fluree/stage (fluree/db ledger) data) (fluree/commit! ledger))) -(defn retry-load - "Retry loading a ledger until max-attempts. Hopefully not needed once JSON-LD - code has an equivalent to :syncTo" - [conn ledger-alias max-attempts] +(defn retry-promise-wrapped + "Retries a fn that when deref'd might return a Throwable. Intended for + retrying promise-wrapped API fns." + [thrower max-attempts] (loop [attempt 0] - (let [ledger (try - (let [res @(fluree/load conn ledger-alias)] - (if (instance? Throwable res) - (throw res) - res)) - (catch Exception e - (when (= (inc attempt) max-attempts) - (throw e) - (Thread/sleep 100))))] - (if ledger - ledger + (let [final (try + (let [res @(thrower)] + (if (instance? Throwable res) + (throw res) + res)) + (catch Throwable t + (when (= (inc attempt) max-attempts) + (throw t) + (Thread/sleep 100))))] + (if final + final (recur (inc attempt)))))) + +(defn retry-load + "Retry loading a ledger until it loads or max-attempts. Hopefully not needed + once JSON-LD code has an equivalent to :syncTo" + [conn ledger-alias max-attempts] + (retry-promise-wrapped #(fluree/load conn ledger-alias) max-attempts)) + +(defn retry-exists? + "Retry calling exists? until it returns true or max-attempts." + [conn ledger-alias max-atttemts] + (retry-promise-wrapped #(fluree/exists? conn ledger-alias) max-atttemts)) From 40010d47b9dc4749e1915a627264c34b52679915 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 14 Feb 2023 12:18:46 -0700 Subject: [PATCH 4/9] Remove obsolete ref flake creation --- src/fluree/db/json_ld/transact.cljc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fluree/db/json_ld/transact.cljc b/src/fluree/db/json_ld/transact.cljc index a8ce80b37..d2ed6bac6 100644 --- a/src/fluree/db/json_ld/transact.cljc +++ b/src/fluree/db/json_ld/transact.cljc @@ -191,8 +191,7 @@ datatype-map (get-in shacl-map [:datatype pid]) property-flakes* (if existing-pid property-flakes - (cond-> (conj property-flakes (flake/create pid const/$iri k const/$xsd:string t true nil)) - ref? (conj (flake/create pid const/$rdf:type const/$iri const/$xsd:anyURI t true nil)))) + (conj property-flakes (flake/create pid const/$iri k const/$xsd:string t true nil))) ;; check-retracts? - a new subject or property don't require checking for flake retractions check-retracts? (or (not new-subj?) existing-pid) flakes* (if retract? From 1dffc9701fa16acb270152928fa72a9481898aba Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 14 Feb 2023 12:25:16 -0700 Subject: [PATCH 5/9] infer integer values to $xsd:long to prevent overflow --- src/fluree/db/datatype.cljc | 2 +- test/fluree/db/query/fql_parse_test.clj | 32 +++++++++++------------ test/fluree/db/query/index_range_test.clj | 22 ++++++++-------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/fluree/db/datatype.cljc b/src/fluree/db/datatype.cljc index 735f3d208..6a84d4940 100644 --- a/src/fluree/db/datatype.cljc +++ b/src/fluree/db/datatype.cljc @@ -87,7 +87,7 @@ [x] (cond (string? x) const/$xsd:string - (integer? x) const/$xsd:integer + (integer? x) const/$xsd:long ; infer to long to prevent overflow (number? x) const/$xsd:decimal (boolean? x) const/$xsd:boolean)) diff --git a/test/fluree/db/query/fql_parse_test.clj b/test/fluree/db/query/fql_parse_test.clj index b563b9d2c..621694df0 100644 --- a/test/fluree/db/query/fql_parse_test.clj +++ b/test/fluree/db/query/fql_parse_test.clj @@ -51,7 +51,7 @@ :spec {:depth 0 :wildcard? true}} (de-recordify-select select))) (is (= [[{::where/var '?s} - {::where/val 1002 ::where/datatype 7} + {::where/val 1002 ::where/datatype 8} {::where/val "Alice" ::where/datatype 1}]] patterns))) (let [vars-query {:select {"?s" ["*"]} @@ -70,7 +70,7 @@ (de-recordify-select select))) (is (= [[{::where/var '?s} {::where/val 1002 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?name}]] patterns))) (let [query {:context {:ex "http://example.org/ns/"} @@ -88,24 +88,24 @@ (de-recordify-select select))) (is (= [[{::where/var '?s} {::where/val 1002 - ::where/datatype 7} + ::where/datatype 8} {::where/val "Cam" ::where/datatype 1}] [{::where/var '?s} {::where/val 1008 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?f}] [{::where/var '?f} {::where/val 1002 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?name}] [{::where/var '?f} {::where/val 1004 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?age}] [{::where/var '?f} {::where/val 1007 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?email}]] patterns))) (testing "class, optional" @@ -120,18 +120,18 @@ (is (= [[:class [{::where/var '?s} {::where/val 200 - ::where/datatype 7} + ::where/datatype 8} {::where/val 1001 ::where/datatype 0}]] [{::where/var '?s} {::where/val 1002 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?name}] [:optional {::where/patterns [[{::where/var '?s} {::where/val 1006 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?favColor}]] ::where/filters {}}]] patterns)))) @@ -147,20 +147,20 @@ (is (= [[:class [{::where/var '?s} {::where/val 200 - ::where/datatype 7} + ::where/datatype 8} {::where/val 1001 ::where/datatype 0}]] [:union [{::where/patterns [[{::where/var '?s} {::where/val 1007 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?email1}]] ::where/filters {}} {::where/patterns [[{::where/var '?s} {::where/val 1003 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?email2}]] ::where/filters {}}]]] patterns)))) @@ -177,16 +177,16 @@ (is (= [[:class [{::where/var '?s} {::where/val 200 - ::where/datatype 7} + ::where/datatype 8} {::where/val 1001 ::where/datatype 0}]] [{::where/var '?s} {::where/val 1004 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?age}] [{::where/var '?s} {::where/val 1002 - ::where/datatype 7} + ::where/datatype 8} {::where/var '?name}]] patterns)))) (testing "group-by, order-by" diff --git a/test/fluree/db/query/index_range_test.clj b/test/fluree/db/query/index_range_test.clj index 3abab6287..b4b742cd2 100644 --- a/test/fluree/db/query/index_range_test.clj +++ b/test/fluree/db/query/index_range_test.clj @@ -49,10 +49,10 @@ [alice-sid 200 1001 0 -1 true nil] [alice-sid 1002 "Alice" 1 -1 true nil] [alice-sid 1003 "alice@example.org" 1 -1 true nil] - [alice-sid 1004 50 7 -1 true nil] - [alice-sid 1005 9 7 -1 true nil] - [alice-sid 1005 42 7 -1 true nil] - [alice-sid 1005 76 7 -1 true nil]] + [alice-sid 1004 50 8 -1 true nil] + [alice-sid 1005 9 8 -1 true nil] + [alice-sid 1005 42 8 -1 true nil] + [alice-sid 1005 76 8 -1 true nil]] (->> @(fluree/slice db :spot [alice-sid]) (mapv flake/Flake->parts))) "Slice should return a vector of flakes for only Alice"))) @@ -60,9 +60,9 @@ (testing "Slice for subject + predicate" (let [alice-sid @(fluree/internal-id db :ex/alice) favNums-pid @(fluree/internal-id db :ex/favNums)] - (is (= [[alice-sid favNums-pid 9 7 -1 true nil] - [alice-sid favNums-pid 42 7 -1 true nil] - [alice-sid favNums-pid 76 7 -1 true nil]] + (is (= [[alice-sid favNums-pid 9 8 -1 true nil] + [alice-sid favNums-pid 42 8 -1 true nil] + [alice-sid favNums-pid 76 8 -1 true nil]] (->> @(fluree/slice db :spot [alice-sid favNums-pid]) (mapv flake/Flake->parts))) "Slice should only return Alice's favNums (multi-cardinality)"))) @@ -70,7 +70,7 @@ (testing "Slice for subject + predicate + value" (let [alice-sid @(fluree/internal-id db :ex/alice) favNums-pid @(fluree/internal-id db :ex/favNums)] - (is (= [[alice-sid favNums-pid 42 7 -1 true nil]] + (is (= [[alice-sid favNums-pid 42 8 -1 true nil]] (->> @(fluree/slice db :spot [alice-sid favNums-pid 42]) (mapv flake/Flake->parts))) "Slice should only return the specified favNum value"))) @@ -78,8 +78,8 @@ (testing "Slice for subject + predicate + value + datatype" (let [alice-sid @(fluree/internal-id db :ex/alice) favNums-pid @(fluree/internal-id db :ex/favNums)] - (is (= [[alice-sid favNums-pid 42 7 -1 true nil]] - (->> @(fluree/slice db :spot [alice-sid favNums-pid [42 7]]) + (is (= [[alice-sid favNums-pid 42 8 -1 true nil]] + (->> @(fluree/slice db :spot [alice-sid favNums-pid [42 8]]) (mapv flake/Flake->parts))) "Slice should only return the specified favNum value with matching datatype"))) @@ -87,7 +87,7 @@ (let [alice-sid @(fluree/internal-id db :ex/alice) favNums-pid @(fluree/internal-id db :ex/favNums)] (is (= [] - (->> @(fluree/slice db :spot [alice-sid favNums-pid [42 8]]) + (->> @(fluree/slice db :spot [alice-sid favNums-pid [42 7]]) (mapv flake/Flake->parts))) "We specify a different datatype for the value, nothing should be returned"))) From 0f50e36317d394af9cd99690c99b73e35036381f Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 14 Feb 2023 12:25:31 -0700 Subject: [PATCH 6/9] Fix a docstring typo --- src/fluree/db/query/exec/select.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/query/exec/select.cljc b/src/fluree/db/query/exec/select.cljc index 5797ddf2e..454df923f 100644 --- a/src/fluree/db/query/exec/select.cljc +++ b/src/fluree/db/query/exec/select.cljc @@ -130,6 +130,6 @@ format-ch)) (defn implicit-grouping? - "Returns true if the provide `selector` can only operate on grouped elements." + "Returns true if the provided `selector` can only operate on grouped elements." [selector] (instance? AggregateSelector selector)) From fa1d4ba63728421a56de46f832d0549ca15cfb6f Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 14 Feb 2023 12:26:39 -0700 Subject: [PATCH 7/9] Delete unused f.d.q.json-ld.response/iri-only-ref fn --- src/fluree/db/query/json_ld/response.cljc | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/fluree/db/query/json_ld/response.cljc b/src/fluree/db/query/json_ld/response.cljc index 0732b5440..03ff6ceeb 100644 --- a/src/fluree/db/query/json_ld/response.cljc +++ b/src/fluree/db/query/json_ld/response.cljc @@ -57,23 +57,6 @@ (vswap! cache assoc sid iri) iri))) - -(defn iri-only-ref - "Extracts result information from a ref predicate. If sub-select exists - and additional graph crawl is performed. If it doesn't exist, simply returns - {@id } for each object." - [db cache compact-fn p-flakes] - (go-try - (let [id-key (:as (wildcard-spec db cache compact-fn const/$iri))] - (loop [[next-flake & r] p-flakes - acc []] - (if next-flake - (let [iri ( Date: Tue, 14 Feb 2023 12:34:03 -0700 Subject: [PATCH 8/9] Improve arg name & docstring of retry-promise-wrapped --- test/fluree/db/test_utils.cljc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/fluree/db/test_utils.cljc b/test/fluree/db/test_utils.cljc index be8560841..b2eecf08c 100644 --- a/test/fluree/db/test_utils.cljc +++ b/test/fluree/db/test_utils.cljc @@ -109,11 +109,12 @@ (defn retry-promise-wrapped "Retries a fn that when deref'd might return a Throwable. Intended for - retrying promise-wrapped API fns." - [thrower max-attempts] + retrying promise-wrapped API fns. Do not deref the return value, this will do + it for you." + [pwrapped max-attempts] (loop [attempt 0] (let [final (try - (let [res @(thrower)] + (let [res @(pwrapped)] (if (instance? Throwable res) (throw res) res)) From 3f72c78585f9a09d898aa191b0d7251aaa3971c2 Mon Sep 17 00:00:00 2001 From: Wes Morgan Date: Tue, 14 Feb 2023 12:40:42 -0700 Subject: [PATCH 9/9] De-parallelize retracts & asserts b/c passing a volatile I did this originally but didn't notice we were passing the same volatile iris map to both retract-flakes and assert-flakes, so probably not safe to parallelize. --- src/fluree/db/json_ld/reify.cljc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/json_ld/reify.cljc b/src/fluree/db/json_ld/reify.cljc index d254165cf..299cabc4d 100644 --- a/src/fluree/db/json_ld/reify.cljc +++ b/src/fluree/db/json_ld/reify.cljc @@ -339,10 +339,10 @@ {:status 500 :error :db/invalid-commit}))) assert (db-assert db-data) retract (db-retract db-data) - retract-flakes (retract-flakes db retract t-new iris) + retract-flakes ( (empty (get-in db [:novelty :spot])) - (into (